JUnit version 4.13.2
.21:00:15.276 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783544415269
21:00:15.328 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
21:00:15.359 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:00:15.369 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasksyq7ka/test-tmp
21:00:15.395 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:00:15.397 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasksyq7ka/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:00:15.597 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:00:15.799 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:00:15.800 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:00:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:15.804286 12350 mini_oidc.cc:150] Starting JWKS server
21:00:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:15.807554 12350 webserver.cc:534] Webserver started at http://127.0.0.1:38471/ using document root <none> and password file <none>
21:00:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:15.807891 12350 mini_oidc.cc:186] Starting OIDC Discovery server
21:00:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:15.809094 12350 webserver.cc:534] Webserver started at http://127.0.0.1:41133/ using document root <none> and password file <none>
21:00:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:00:15Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:00:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:00:15Z Disabled control of system clock
21:00:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:15.844463 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal
21:00:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data
21:00:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/logs
21:00:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data/info.pb
21:00:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.190:41363
21:00:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.190
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38471/jwks.json with env {}
21:00:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.175663 12361 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.175972 12361 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.176051 12361 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.184888 12361 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:00:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.184986 12361 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.185047 12361 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:00:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.185088 12361 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:00:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.185170 12361 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:00:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.185222 12361 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38471/jwks.json
21:00:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.195727 12361 master_runner.cc:387] Master server non-default flags:
21:00:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data
21:00:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal
21:00:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.190:41363
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38471/jwks.json
21:00:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data/info.pb
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.190
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/logs
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.197892 12361 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.200163 12361 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.209483 12367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.210350 12366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:16.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.213833 12361 server_base.cc:1061] running on GCE node
21:00:16.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.214651 12369 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:16.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.215703 12361 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.217381 12361 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:16.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.220842 12361 hybrid_clock.cc:648] HybridClock initialized: now 1783544416220413 us; error 424 us; skew 500 ppm
21:00:16.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.221364 12361 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:16.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.224262 12361 webserver.cc:534] Webserver started at http://127.12.15.190:41101/ using document root <none> and password file <none>
21:00:16.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.225206 12361 fs_manager.cc:362] Metadata directory not provided
21:00:16.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.225370 12361 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.225845 12361 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:00:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.228560 12361 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data/instance:
21:00:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "295fa41dde4d4ea484019f911c0858d2"
21:00:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:16 on dist-test-slave-9w0b"
21:00:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.229514 12361 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal/instance:
21:00:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "295fa41dde4d4ea484019f911c0858d2"
21:00:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:16 on dist-test-slave-9w0b"
21:00:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.235487 12361 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.003s	sys 0.005s
21:00:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.239320 12375 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.240845 12361 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.002s
21:00:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.241048 12361 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal
21:00:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "295fa41dde4d4ea484019f911c0858d2"
21:00:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:16 on dist-test-slave-9w0b"
21:00:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.241305 12361 fs_report.cc:389] FS layout report
21:00:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal
21:00:16.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal
21:00:16.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data/data
21:00:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.276032 12361 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.277179 12361 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.277531 12361 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:16.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.299113 12361 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.190:41363
21:00:16.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.299136 12426 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.190:41363 every 8 connection(s)
21:00:16.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.301028 12361 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/data/info.pb
21:00:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.305037 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 12361
21:00:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.305317 12350 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-0/wal/instance
21:00:16.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.305967 12427 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.307682 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal
21:00:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data
21:00:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/logs
21:00:16.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data/info.pb
21:00:16.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:16.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:16.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:16.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.189:37857
21:00:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.189
21:00:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38471/jwks.json with env {}
21:00:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.314610 12427 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } has no permanent_uuid. Determining permanent_uuid...
21:00:16.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.340258 12427 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } has no permanent_uuid. Determining permanent_uuid...
21:00:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.341426 12376 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.189:37857: connect: Connection refused (error 111)
21:00:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.343176 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.189:37857: Network error: Client connection negotiation failed: client connection to 127.12.15.189:37857: connect: Connection refused (error 111)
21:00:16.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.392581 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } attempt: 1
21:00:16.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.394932 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.189:37857: Network error: Client connection negotiation failed: client connection to 127.12.15.189:37857: connect: Connection refused (error 111)
21:00:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.463260 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } attempt: 2
21:00:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.465549 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.189:37857: Network error: Client connection negotiation failed: client connection to 127.12.15.189:37857: connect: Connection refused (error 111)
21:00:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.556834 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } attempt: 3
21:00:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.559427 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.189:37857: Network error: Client connection negotiation failed: client connection to 127.12.15.189:37857: connect: Connection refused (error 111)
21:00:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.663066 12429 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:16.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.663463 12429 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:16.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.663571 12429 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.673496 12429 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:00:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.673622 12429 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.673686 12429 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:00:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.673731 12429 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:00:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.673789 12429 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:00:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.673828 12429 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38471/jwks.json
21:00:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.684608 12429 master_runner.cc:387] Master server non-default flags:
21:00:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.189:37857
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38471/jwks.json
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data/info.pb
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.189
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/logs
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.686836 12429 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.689251 12429 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.698829 12438 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.698918 12437 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.701148 12440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.702173 12429 server_base.cc:1061] running on GCE node
21:00:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.702654 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } attempt: 4
21:00:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.703158 12429 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:16.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.704953 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.189:37857: Network error: Client connection negotiation failed: client connection to 127.12.15.189:37857: connect: Connection refused (error 111)
21:00:16.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.705317 12429 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.706557 12429 hybrid_clock.cc:648] HybridClock initialized: now 1783544416706439 us; error 109 us; skew 500 ppm
21:00:16.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.707065 12429 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.709731 12429 webserver.cc:534] Webserver started at http://127.12.15.189:41185/ using document root <none> and password file <none>
21:00:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.710567 12429 fs_manager.cc:362] Metadata directory not provided
21:00:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.710677 12429 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.711166 12429 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:00:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.714493 12429 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data/instance:
21:00:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd171405dde14cd9ac596f418428baa2"
21:00:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:16 on dist-test-slave-9w0b"
21:00:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.715338 12429 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal/instance:
21:00:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd171405dde14cd9ac596f418428baa2"
21:00:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:16 on dist-test-slave-9w0b"
21:00:16.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.721222 12429 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
21:00:16.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.725378 12446 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.727109 12429 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.727279 12429 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal
21:00:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd171405dde14cd9ac596f418428baa2"
21:00:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:16 on dist-test-slave-9w0b"
21:00:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.727465 12429 fs_report.cc:389] FS layout report
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data/data
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.776027 12429 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.777086 12429 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.777410 12429 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:16.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.799947 12497 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.189:37857 every 8 connection(s)
21:00:16.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.799975 12429 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.189:37857
21:00:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.802103 12429 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/data/info.pb
21:00:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.807082 12498 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:16.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.807868 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 12429
21:00:16.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.808050 12350 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-1/wal/instance
21:00:16.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.809223 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:16.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:16.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal
21:00:16.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/logs
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data/info.pb
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.188:37439
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.188
21:00:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38471/jwks.json with env {}
21:00:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.815325 12498 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } has no permanent_uuid. Determining permanent_uuid...
21:00:16.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.833089 12498 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } has no permanent_uuid. Determining permanent_uuid...
21:00:16.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.844465 12498 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } has no permanent_uuid. Determining permanent_uuid...
21:00:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.845677 12447 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.847378 12498 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.896842 12498 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 1
21:00:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.899228 12498 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:16.967509 12498 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 2
21:00:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:16.970186 12498 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.004282 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } attempt: 5
21:00:17.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.012432 12427 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } has no permanent_uuid. Determining permanent_uuid...
21:00:17.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.014590 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.061466 12498 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 3
21:00:17.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.063982 12498 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.065876 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 1
21:00:17.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.068285 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.136638 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 2
21:00:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.139115 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.165426 12500 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.165776 12500 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.165859 12500 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.174494 12500 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:00:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.174609 12500 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.174665 12500 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:00:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.174775 12500 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:00:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.174847 12500 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:00:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.174882 12500 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38471/jwks.json
21:00:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.185433 12500 master_runner.cc:387] Master server non-default flags:
21:00:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.188:37439
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38471/jwks.json
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data/info.pb
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.188
21:00:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/logs
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.187386 12500 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.189841 12500 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.199282 12509 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.199371 12508 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.200457 12511 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:17.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.200970 12500 server_base.cc:1061] running on GCE node
21:00:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.201969 12500 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:17.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.203696 12500 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.204923 12500 hybrid_clock.cc:648] HybridClock initialized: now 1783544417204858 us; error 61 us; skew 500 ppm
21:00:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.205264 12500 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.207226 12498 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 4
21:00:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.208379 12500 webserver.cc:534] Webserver started at http://127.12.15.188:33817/ using document root <none> and password file <none>
21:00:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.209371 12500 fs_manager.cc:362] Metadata directory not provided
21:00:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.209471 12500 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:17.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.209916 12500 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:00:17.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.210032 12498 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.212517 12500 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data/instance:
21:00:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfae842cb6034e4c88dce26814d31218"
21:00:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:17 on dist-test-slave-9w0b"
21:00:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.213277 12500 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal/instance:
21:00:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfae842cb6034e4c88dce26814d31218"
21:00:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:17 on dist-test-slave-9w0b"
21:00:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.219420 12500 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.003s	sys 0.004s
21:00:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.224309 12517 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:17.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.226225 12500 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
21:00:17.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.226433 12500 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfae842cb6034e4c88dce26814d31218"
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:17 on dist-test-slave-9w0b"
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.226727 12500 fs_report.cc:389] FS layout report
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data/data
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.243065 12500 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.243932 12500 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.244175 12500 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.245347 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 3
21:00:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.247689 12427 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.15.188:37439: Network error: Client connection negotiation failed: client connection to 127.12.15.188:37439: connect: Connection refused (error 111)
21:00:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.266075 12500 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.188:37439
21:00:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.266129 12568 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.188:37439 every 8 connection(s)
21:00:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.267903 12500 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/data/info.pb
21:00:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.269572 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 12500
21:00:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.269776 12350 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/master-2/wal/instance
21:00:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.274011 12569 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.281287 12569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } has no permanent_uuid. Determining permanent_uuid...
21:00:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.296906 12569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } has no permanent_uuid. Determining permanent_uuid...
21:00:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.305171 12569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } has no permanent_uuid. Determining permanent_uuid...
21:00:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.317557 12569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: Bootstrap starting.
21:00:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.322100 12569 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: Neither blocks nor log segments found. Creating new log.
21:00:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.323436 12569 log.cc:826] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: Log is configured to *not* fsync() on all Append() calls
21:00:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.326779 12569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: No bootstrap required, opened a new log
21:00:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.332463 12569 raft_consensus.cc:359] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.332839 12569 raft_consensus.cc:385] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.332970 12569 raft_consensus.cc:740] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bfae842cb6034e4c88dce26814d31218, State: Initialized, Role: FOLLOWER
21:00:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.333680 12569 consensus_queue.cc:260] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.334618 12576 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.334959 12576 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.335750 12569 sys_catalog.cc:565] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: configured and running, proceeding with master startup.
21:00:17.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.347069 12587 catalog_manager.cc:1593] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:00:17.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.347227 12587 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:00:17.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.343591 12500 master_runner.cc:187] Error getting master registration for 127.12.15.190:41363: OK, instance_id { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" instance_seqno: 1783544416293374 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:00:17.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.348527 12500 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:00:17.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.424971 12427 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 4
21:00:17.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.436657 12427 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: Bootstrap starting.
21:00:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.440291 12427 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: Neither blocks nor log segments found. Creating new log.
21:00:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.441437 12427 log.cc:826] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: Log is configured to *not* fsync() on all Append() calls
21:00:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.444164 12427 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: No bootstrap required, opened a new log
21:00:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.450660 12427 raft_consensus.cc:359] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.451025 12427 raft_consensus.cc:385] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.451121 12427 raft_consensus.cc:740] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 295fa41dde4d4ea484019f911c0858d2, State: Initialized, Role: FOLLOWER
21:00:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.451766 12427 consensus_queue.cc:260] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.452760 12590 sys_catalog.cc:455] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.453056 12590 sys_catalog.cc:458] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.454097 12427 sys_catalog.cc:565] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: configured and running, proceeding with master startup.
21:00:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.464774 12601 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:00:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.465417 12601 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:00:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.464301 12361 master_runner.cc:187] Error getting master registration for 127.12.15.189:37857: OK, instance_id { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" instance_seqno: 1783544416794138 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:00:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.474767 12361 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:00:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.497056 12523 tablet.cc:2406] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:17.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.509280 12498 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } attempt: 5
21:00:17.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.521046 12498 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: Bootstrap starting.
21:00:17.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.524487 12498 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: Neither blocks nor log segments found. Creating new log.
21:00:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.525681 12498 log.cc:826] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: Log is configured to *not* fsync() on all Append() calls
21:00:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.528396 12498 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: No bootstrap required, opened a new log
21:00:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.530776 12381 tablet.cc:2406] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.536648 12498 raft_consensus.cc:359] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.537030 12498 raft_consensus.cc:385] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.537168 12498 raft_consensus.cc:740] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd171405dde14cd9ac596f418428baa2, State: Initialized, Role: FOLLOWER
21:00:17.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.538002 12498 consensus_queue.cc:260] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.539160 12604 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.539819 12604 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.540643 12498 sys_catalog.cc:565] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: configured and running, proceeding with master startup.
21:00:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.562250 12429 master_runner.cc:264] No leader master found from master fd171405dde14cd9ac596f418428baa2
21:00:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.562459 12429 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:00:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.565119 12615 catalog_manager.cc:1593] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:00:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.565294 12615 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:00:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.730288 12576 raft_consensus.cc:493] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.730621 12576 raft_consensus.cc:515] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.732656 12576 leader_election.cc:290] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 295fa41dde4d4ea484019f911c0858d2 (127.12.15.190:41363), fd171405dde14cd9ac596f418428baa2 (127.12.15.189:37857)
21:00:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.733938 12473 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfae842cb6034e4c88dce26814d31218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd171405dde14cd9ac596f418428baa2" is_pre_election: true
21:00:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.734495 12473 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bfae842cb6034e4c88dce26814d31218 in term 0.
21:00:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.735260 12518 leader_election.cc:304] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bfae842cb6034e4c88dce26814d31218, fd171405dde14cd9ac596f418428baa2; no voters: 
21:00:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.737285 12576 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:00:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.737461 12576 raft_consensus.cc:493] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.737556 12576 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 0 FOLLOWER]: Advancing to term 1
21:00:17.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.740355 12402 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfae842cb6034e4c88dce26814d31218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "295fa41dde4d4ea484019f911c0858d2" is_pre_election: true
21:00:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.740963 12402 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bfae842cb6034e4c88dce26814d31218 in term 0.
21:00:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.758045 12576 raft_consensus.cc:515] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.760627 12402 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfae842cb6034e4c88dce26814d31218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "295fa41dde4d4ea484019f911c0858d2"
21:00:17.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.760891 12402 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 0 FOLLOWER]: Advancing to term 1
21:00:17.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.765051 12402 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bfae842cb6034e4c88dce26814d31218 in term 1.
21:00:17.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.765774 12519 leader_election.cc:304] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 295fa41dde4d4ea484019f911c0858d2, bfae842cb6034e4c88dce26814d31218; no voters: 
21:00:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.771144 12473 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfae842cb6034e4c88dce26814d31218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd171405dde14cd9ac596f418428baa2"
21:00:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.771417 12473 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 0 FOLLOWER]: Advancing to term 1
21:00:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.775169 12473 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bfae842cb6034e4c88dce26814d31218 in term 1.
21:00:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.776989 12576 leader_election.cc:290] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [CANDIDATE]: Term 1 election: Requested vote from peers 295fa41dde4d4ea484019f911c0858d2 (127.12.15.190:41363), fd171405dde14cd9ac596f418428baa2 (127.12.15.189:37857)
21:00:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.777313 12576 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 1 FOLLOWER]: Leader election won for term 1
21:00:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.777447 12576 raft_consensus.cc:697] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [term 1 LEADER]: Becoming Leader. State: Replica: bfae842cb6034e4c88dce26814d31218, State: Running, Role: LEADER
21:00:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.778017 12576 consensus_queue.cc:237] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } }
21:00:17.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:17.780839 12452 tablet.cc:2406] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.804385 12617 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfae842cb6034e4c88dce26814d31218. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.805364 12617 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: This master's current role is: LEADER
21:00:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.829703 12619 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
21:00:17.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.832758 12619 catalog_manager.cc:1519] Initializing Kudu cluster ID...
21:00:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.855465 12473 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [term 1 FOLLOWER]: Refusing update from remote peer bfae842cb6034e4c88dce26814d31218: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:00:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.855767 12402 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [term 1 FOLLOWER]: Refusing update from remote peer bfae842cb6034e4c88dce26814d31218: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:00:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.856971 12617 consensus_queue.cc:1048] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [LEADER]: Connected to new peer: Peer: permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.857301 12576 consensus_queue.cc:1048] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.881349 12590 sys_catalog.cc:455] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfae842cb6034e4c88dce26814d31218. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.881693 12590 sys_catalog.cc:458] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.885600 12620 mvcc.cc:204] Tried to move back new op lower bound from 7305397935523540992 to 7305397935255977984. Current Snapshot: MvccSnapshot[applied={T|T < 7305397935523540992}]
21:00:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.889101 12622 mvcc.cc:204] Tried to move back new op lower bound from 7305397935523540992 to 7305397935255977984. Current Snapshot: MvccSnapshot[applied={T|T < 7305397935523540992}]
21:00:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.895121 12590 sys_catalog.cc:455] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.895402 12590 sys_catalog.cc:458] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.901638 12617 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.901887 12617 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: This master's current role is: LEADER
21:00:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.903347 12619 catalog_manager.cc:1382] Generated new cluster ID: 8ad254b5b0554a35a4ceeff005ad9faf
21:00:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.903475 12619 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
21:00:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.903775 12604 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfae842cb6034e4c88dce26814d31218. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.904035 12604 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.911747 12621 mvcc.cc:204] Tried to move back new op lower bound from 7305397935523540992 to 7305397935255977984. Current Snapshot: MvccSnapshot[applied={T|T < 7305397935523540992}]
21:00:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.919155 12576 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.919847 12576 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218 [sys.catalog]: This master's current role is: LEADER
21:00:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.942144 12604 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bfae842cb6034e4c88dce26814d31218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "295fa41dde4d4ea484019f911c0858d2" member_type: VOTER last_known_addr { host: "127.12.15.190" port: 41363 } } peers { permanent_uuid: "fd171405dde14cd9ac596f418428baa2" member_type: VOTER last_known_addr { host: "127.12.15.189" port: 37857 } } peers { permanent_uuid: "bfae842cb6034e4c88dce26814d31218" member_type: VOTER last_known_addr { host: "127.12.15.188" port: 37439 } } }
21:00:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.942451 12604 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2 [sys.catalog]: This master's current role is: FOLLOWER
21:00:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.954171 12619 catalog_manager.cc:1405] Generated new certificate authority record
21:00:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.955935 12619 catalog_manager.cc:1539] Loading token signing keys...
21:00:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.976909 12619 catalog_manager.cc:6069] T 00000000000000000000000000000000 P bfae842cb6034e4c88dce26814d31218: Generated new TSK 0
21:00:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:17.978642 12619 catalog_manager.cc:1549] Initializing in-progress tserver states...
21:00:18.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.015571 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:0
21:00:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.455916 12631 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.456394 12631 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.456511 12631 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.465164 12631 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.465361 12631 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.129
21:00:18.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.468694 12601 catalog_manager.cc:1294] Loaded cluster ID: 8ad254b5b0554a35a4ceeff005ad9faf
21:00:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.468834 12601 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: loading cluster ID for follower catalog manager: success
21:00:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.471781 12601 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: acquiring CA information for follower catalog manager: success
21:00:18.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.474406 12601 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 295fa41dde4d4ea484019f911c0858d2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:00:18.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.477418 12631 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:18.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:0
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.479245 12631 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.481295 12631 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:18.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.491765 12636 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:18.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.491837 12637 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:18.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.494344 12639 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.495033 12631 server_base.cc:1061] running on GCE node
21:00:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.495707 12631 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.496779 12631 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:18.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.498136 12631 hybrid_clock.cc:648] HybridClock initialized: now 1783544418498014 us; error 107 us; skew 500 ppm
21:00:18.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.498569 12631 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:18.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.501292 12631 webserver.cc:534] Webserver started at http://127.12.15.129:42207/ using document root <none> and password file <none>
21:00:18.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.502202 12631 fs_manager.cc:362] Metadata directory not provided
21:00:18.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.502367 12631 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:18.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.502866 12631 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:00:18.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.505390 12631 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/instance:
21:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.506330 12631 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal/instance:
21:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.512449 12631 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.004s	sys 0.001s
21:00:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.516361 12645 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.517961 12631 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
21:00:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.518160 12631 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.518445 12631 fs_report.cc:389] FS layout report
21:00:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/data
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:18.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.542997 12631 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:18.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.544060 12631 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:18.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.544402 12631 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:18.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.545581 12631 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:18.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.547457 12631 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:00:18.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.547593 12631 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:00:18.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.547714 12631 ts_tablet_manager.cc:616] Registered 0 tablets
21:00:18.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.547793 12631 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:00:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.567822 12615 catalog_manager.cc:1294] Loaded cluster ID: 8ad254b5b0554a35a4ceeff005ad9faf
21:00:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.567965 12615 catalog_manager.cc:1587] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: loading cluster ID for follower catalog manager: success
21:00:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.570827 12615 catalog_manager.cc:1609] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: acquiring CA information for follower catalog manager: success
21:00:18.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.573295 12615 catalog_manager.cc:1637] T 00000000000000000000000000000000 P fd171405dde14cd9ac596f418428baa2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:00:18.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.586838 12631 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.129:46601
21:00:18.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.586860 12757 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.129:46601 every 8 connection(s)
21:00:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.588425 12631 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.589203 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 12631
21:00:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.589439 12350 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal/instance
21:00:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.595809 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:0
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.622792 12758 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.623485 12758 heartbeater.cc:461] Registering TS with master...
21:00:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.624712 12758 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.624748 12759 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:18.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.624974 12759 heartbeater.cc:461] Registering TS with master...
21:00:18.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.625559 12759 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.627691 12534 ts_manager.cc:194] Registered new tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.627688 12392 ts_manager.cc:194] Registered new tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.629375 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.129:52537
21:00:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.629498 12760 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.629621 12760 heartbeater.cc:461] Registering TS with master...
21:00:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.630089 12760 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.632225 12463 ts_manager.cc:194] Registered new tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.960692 12768 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.961062 12768 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.961177 12768 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.969720 12768 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:18.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.969959 12768 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.130
21:00:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.980432 12768 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:0
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.982796 12768 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.984853 12768 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:18.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.995023 12774 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:18.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.995023 12773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:18.995313 12776 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.996172 12768 server_base.cc:1061] running on GCE node
21:00:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.996858 12768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.998000 12768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.999236 12768 hybrid_clock.cc:648] HybridClock initialized: now 1783544418999164 us; error 70 us; skew 500 ppm
21:00:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:18.999619 12768 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:19.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.003378 12768 webserver.cc:534] Webserver started at http://127.12.15.130:42015/ using document root <none> and password file <none>
21:00:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.004276 12768 fs_manager.cc:362] Metadata directory not provided
21:00:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.004417 12768 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.004841 12768 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:00:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.007467 12768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/instance:
21:00:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.008421 12768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal/instance:
21:00:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.014238 12768 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.001s	sys 0.005s
21:00:19.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.017983 12782 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:19.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.019516 12768 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:19.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.019660 12768 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:19.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:19.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.019879 12768 fs_report.cc:389] FS layout report
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/data
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.035656 12768 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.036584 12768 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.036829 12768 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:19.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.038002 12768 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.039839 12768 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:00:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.039932 12768 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:00:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.040005 12768 ts_tablet_manager.cc:616] Registered 0 tablets
21:00:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.040055 12768 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:00:19.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.074378 12768 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.130:39631
21:00:19.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.074427 12894 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.130:39631 every 8 connection(s)
21:00:19.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.075887 12768 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.081519 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 12768
21:00:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.081794 12350 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal/instance
21:00:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.096433 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:0
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.104888 12895 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.105366 12895 heartbeater.cc:461] Registering TS with master...
21:00:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.105988 12896 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.106194 12896 heartbeater.cc:461] Registering TS with master...
21:00:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.106535 12895 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.106770 12897 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.106886 12897 heartbeater.cc:461] Registering TS with master...
21:00:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.107338 12897 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.108199 12896 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.109673 12534 ts_manager.cc:194] Registered new tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.110934 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.130:54617
21:00:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.112463 12392 ts_manager.cc:194] Registered new tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.112412 12463 ts_manager.cc:194] Registered new tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.451627 12902 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.451924 12902 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.452001 12902 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.460337 12902 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.460506 12902 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.131
21:00:19.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.470590 12902 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:0
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:19.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:19.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.472501 12902 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:19.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.474886 12902 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:19.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.487656 12908 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:19.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.487685 12907 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:19.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.488960 12902 server_base.cc:1061] running on GCE node
21:00:19.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:19.488207 12910 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:19.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.489944 12902 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:19.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.491145 12902 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:19.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.492449 12902 hybrid_clock.cc:648] HybridClock initialized: now 1783544419492393 us; error 50 us; skew 500 ppm
21:00:19.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.492980 12902 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.495581 12902 webserver.cc:534] Webserver started at http://127.12.15.131:36255/ using document root <none> and password file <none>
21:00:19.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.496430 12902 fs_manager.cc:362] Metadata directory not provided
21:00:19.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.496572 12902 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.497020 12902 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:00:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.502902 12902 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/instance:
21:00:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:19 on dist-test-slave-9w0b"
21:00:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.504081 12902 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal/instance:
21:00:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:19 on dist-test-slave-9w0b"
21:00:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.509974 12902 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.002s	sys 0.004s
21:00:19.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.513892 12916 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:19.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.515591 12902 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.515863 12902 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:19 on dist-test-slave-9w0b"
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.516162 12902 fs_report.cc:389] FS layout report
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/data
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:19.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.532687 12902 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.533816 12902 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.534195 12902 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.535580 12902 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:19.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.537508 12902 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:00:19.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.537647 12902 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:00:19.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.537760 12902 ts_tablet_manager.cc:616] Registered 0 tablets
21:00:19.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.537842 12902 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:00:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.575783 12902 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.131:42675
21:00:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.575824 13028 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.131:42675 every 8 connection(s)
21:00:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.577394 12902 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.584654 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 12902
21:00:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.584828 12350 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal/instance
21:00:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.596118 13031 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.596314 13030 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.596534 13030 heartbeater.cc:461] Registering TS with master...
21:00:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.596541 13031 heartbeater.cc:461] Registering TS with master...
21:00:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.597777 13031 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.597905 13030 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.599673 13029 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.599711 12392 ts_manager.cc:194] Registered new tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.599807 13029 heartbeater.cc:461] Registering TS with master...
21:00:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.599790 12463 ts_manager.cc:194] Registered new tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.600317 13029 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.601254 12534 ts_manager.cc:194] Registered new tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.602022 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.131:54057
21:00:19.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.610939 12350 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
21:00:19.613 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:00:19.614 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

21:00:19.624 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.12.15.190"
      port: 41363
    }
    bound_http_address {
      host: "127.12.15.190"
      port: 41101
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.12.15.189"
      port: 37857
    }
    bound_http_address {
      host: "127.12.15.189"
      port: 41185
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.12.15.188"
      port: 37439
    }
    bound_http_address {
      host: "127.12.15.188"
      port: 33817
    }
  }
}

21:00:19.628 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:00:19.631 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.12.15.129"
      port: 46601
    }
    bound_http_address {
      host: "127.12.15.129"
      port: 42207
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.12.15.130"
      port: 39631
    }
    bound_http_address {
      host: "127.12.15.130"
      port: 42015
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.12.15.131"
      port: 42675
    }
    bound_http_address {
      host: "127.12.15.131"
      port: 36255
    }
  }
}

21:00:19.631 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:00:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:19.632577 12758 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:19.803 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:00:19.868 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
21:00:20.052 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.15.189:37857(127.12.15.189:37857)] connecting to peer
21:00:20.108 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.15.188:37439(127.12.15.188:37439)] connecting to peer
21:00:20.109 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.15.190:41363(127.12.15.190:41363)] connecting to peer
21:00:20.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.114511 12895 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:20.134 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.12.15.190:41363(127.12.15.190:41363)] Successfully connected to peer
21:00:20.137 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.15.188:37439(127.12.15.188:37439)] Successfully connected to peer
21:00:20.137 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.15.189:37857(127.12.15.189:37857)] Successfully connected to peer
21:00:20.231 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.15.188
21:00:20.231 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.15.189
21:00:20.231 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.15.190
21:00:20.457 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5094454a, L:/127.0.0.1:44084 - R:/127.12.15.189:37857] using SASL/PLAIN
21:00:20.457 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xe80d05ec, L:/127.0.0.1:40336 - R:/127.12.15.190:41363] using SASL/PLAIN
21:00:20.457 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xff192e9d, L:/127.0.0.1:60148 - R:/127.12.15.188:37439] using SASL/PLAIN
21:00:20.490 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:00:20.491 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=487), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.15.189:37857, [98ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.15.188:37439, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.15.190:41363, [461ms] Sub RPC ConnectToMaster: received response from server master-127.12.15.189:37857: OK, [461ms] Sub RPC ConnectToMaster: received response from server master-127.12.15.190:41363: OK, [461ms] Sub RPC ConnectToMaster: received response from server master-127.12.15.188:37439: OK, deferred=null) after lookup
21:00:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.541152 12534 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60148:
21:00:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987"
21:00:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:00:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:20.543510 12534 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:00:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.604266 13029 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.625937 12693 tablet_service.cc:1511] Processing CreateTablet for tablet d1bb5190405140dfa1141ac6f4c37eb3 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987 [id=69189436124c4923b774360e05720948]), partition=RANGE (key) PARTITION UNBOUNDED
21:00:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.628242 12693 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d1bb5190405140dfa1141ac6f4c37eb3. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:20.633 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 69189436124c4923b774360e05720948
21:00:20.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.650823 12964 tablet_service.cc:1511] Processing CreateTablet for tablet d1bb5190405140dfa1141ac6f4c37eb3 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987 [id=69189436124c4923b774360e05720948]), partition=RANGE (key) PARTITION UNBOUNDED
21:00:20.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.652773 12964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d1bb5190405140dfa1141ac6f4c37eb3. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:20.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.652329 12830 tablet_service.cc:1511] Processing CreateTablet for tablet d1bb5190405140dfa1141ac6f4c37eb3 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987 [id=69189436124c4923b774360e05720948]), partition=RANGE (key) PARTITION UNBOUNDED
21:00:20.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.653729 12830 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d1bb5190405140dfa1141ac6f4c37eb3. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.661106 13061 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap starting.
21:00:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.665221 13061 tablet_bootstrap.cc:654] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Neither blocks nor log segments found. Creating new log.
21:00:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.669028 13061 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Log is configured to *not* fsync() on all Append() calls
21:00:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.670054 13063 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap starting.
21:00:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.673429 13062 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap starting.
21:00:20.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.676589 13063 tablet_bootstrap.cc:654] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Neither blocks nor log segments found. Creating new log.
21:00:20.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.676651 13062 tablet_bootstrap.cc:654] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Neither blocks nor log segments found. Creating new log.
21:00:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.676981 13061 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: No bootstrap required, opened a new log
21:00:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.677295 13061 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent bootstrapping tablet: real 0.017s	user 0.010s	sys 0.000s
21:00:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.677838 13062 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Log is configured to *not* fsync() on all Append() calls
21:00:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.677894 13063 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Log is configured to *not* fsync() on all Append() calls
21:00:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.687026 13062 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: No bootstrap required, opened a new log
21:00:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.687026 13063 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: No bootstrap required, opened a new log
21:00:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.687352 13063 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent bootstrapping tablet: real 0.018s	user 0.010s	sys 0.004s
21:00:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.687346 13061 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.687964 13061 raft_consensus.cc:385] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.688015 13062 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent bootstrapping tablet: real 0.015s	user 0.006s	sys 0.007s
21:00:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.688338 13061 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Initialized, Role: FOLLOWER
21:00:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.691515 13061 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.695770 13061 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent starting tablet: real 0.018s	user 0.016s	sys 0.000s
21:00:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.696225 13063 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.697691 13062 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.698073 13062 raft_consensus.cc:385] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:20.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.698173 13062 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Initialized, Role: FOLLOWER
21:00:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.698904 13062 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.700698 13062 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent starting tablet: real 0.013s	user 0.004s	sys 0.004s
21:00:20.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.701905 13063 raft_consensus.cc:385] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:20.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.702224 13063 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Initialized, Role: FOLLOWER
21:00:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.703060 13063 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.706849 13063 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent starting tablet: real 0.019s	user 0.011s	sys 0.001s
21:00:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:20.829630 12898 tablet.cc:2406] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:20.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:20.830459 13032 tablet.cc:2406] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:20.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:20.854218 12764 tablet.cc:2406] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.985028 13067 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.985311 13067 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.987825 13067 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.996090 12850 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165" is_pre_election: true
21:00:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.996661 12850 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 0.
21:00:20.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.996767 12984 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:20.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.997252 12984 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 0.
21:00:20.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.997470 12648 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 
21:00:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.997956 13067 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:00:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.998106 13067 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:20.998219 13067 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 0 FOLLOWER]: Advancing to term 1
21:00:21.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.000384 13067 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.001017 13067 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 1 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.001586 12850 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.001595 12984 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.001776 12984 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 0 FOLLOWER]: Advancing to term 1
21:00:21.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.001797 12850 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 0 FOLLOWER]: Advancing to term 1
21:00:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.004014 12984 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 1.
21:00:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.004087 12850 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 1.
21:00:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.004565 12648 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 
21:00:21.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.004892 13067 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Leader election won for term 1
21:00:21.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.005407 13067 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 LEADER]: Becoming Leader. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Running, Role: LEADER
21:00:21.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.006451 13067 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:21.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.013459 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129). New cstate: current_term: 1 leader_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.478488 13067 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:21.488265 13067 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:21.754 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 69189436124c4923b774360e05720948
21:00:21.772 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:21.780 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:21.782 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=null) after lookup
21:00:21.794 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:21.797 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] Successfully connected to peer
21:00:21.869 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xb7c2b113, L:/127.0.0.1:51650 - R:/127.12.15.129:46601] using TOKEN/null
21:00:22.017 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:22.019 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.129:46601
21:00:22.020 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:22.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:22.021143 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 12631
21:00:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:22.049845 12753 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:22.158825 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 12631
21:00:22.175 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:22.177 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] handling channelInactive
21:00:22.178 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state READY due to: connection closed
21:00:22.178 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: connection closed
21:00:22.189 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:22.190 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:22.190 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [13ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1999048121(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@32a187f5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f75e964 -> wakeup thread Time-limited test)) after lookup
21:00:22.191 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:22.193 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.194 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.194 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:22.209 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:22.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [13ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [15ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [18ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [18ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1999048121(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@32a187f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f75e964 -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368))) after lookup
21:00:22.210 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:22.212 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.212 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.212 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.250 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:22.250 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:22.251 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [13ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [15ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [18ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [18ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [34ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [36ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [36ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [74ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1999048121(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@32a187f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f75e964 -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239))) after lookup
21:00:22.252 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:22.254 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.254 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.254 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.390 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:22.391 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:22.391 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=215), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [13ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [15ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [18ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [18ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [34ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [36ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [36ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [74ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [76ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [78ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [214ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1999048121(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@32a187f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239) -> (continuation of Deferred@1919228625 after retry RPC@1006536539), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f75e964 -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239) -> (continuation of Deferred@1919228625 after retry RPC@1006536539))) after lookup
21:00:22.392 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:22.394 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.394 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.394 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.550 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:22.551 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:22.551 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=375), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [13ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [15ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [18ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [18ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [34ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [36ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [36ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [74ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [76ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [78ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [214ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [216ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [218ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [218ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [369ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [374ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1999048121(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@32a187f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239) -> (continuation of Deferred@1919228625 after retry RPC@1006536539) -> (continuation of Deferred@2087664384 after retry RPC@650789379), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f75e964 -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239) -> (continuation of Deferred@1919228625 after retry RPC@1006536539) -> (continuation of Deferred@2087664384 after retry RPC@650789379))) after lookup
21:00:22.552 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:22.553 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.553 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:22.553 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.449559 13098 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.449602 13099 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.449776 13099 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.449822 13098 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.451364 13099 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.451596 13098 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:23.467919 12783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:23.475992 12917 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:23.477080 12783 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.479460 12984 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:23.479813 12917 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.479890 12984 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 1.
21:00:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.480506 12785 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.480968 13098 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:00:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.481137 13098 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.481271 13098 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 1 FOLLOWER]: Advancing to term 2
21:00:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.482847 12849 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165" is_pre_election: true
21:00:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.484189 13098 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.484675 12849 raft_consensus.cc:2393] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 8152e70912024c1f84634d5e966b4c84 in current term 2: Already voted for candidate 5d6f60d583a5412c84b6412358b50165 in this term.
21:00:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.485164 13098 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 2 election: Requested vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.485836 12984 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.486115 12984 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 1 FOLLOWER]: Advancing to term 2
21:00:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.486142 12919 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84; no voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:23.487110 12783 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:23.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.488677 12984 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 2.
21:00:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.488929 13099 raft_consensus.cc:2749] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:00:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.489188 12785 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.489547 13098 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Leader election won for term 2
21:00:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.490059 13098 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 LEADER]: Becoming Leader. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Running, Role: LEADER
21:00:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.490748 13098 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:23.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:23.495990 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 reported cstate change: term changed from 1 to 2, leader changed from 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129) to 5d6f60d583a5412c84b6412358b50165 (127.12.15.130). New cstate: current_term: 2 leader_uuid: "5d6f60d583a5412c84b6412358b50165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.044214 12783 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 -> Peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Couldn't send request to peer 83ca46c2c5f54616bd88be0a9bb0a7b6. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:24.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.049986 12984 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Refusing update from remote peer 5d6f60d583a5412c84b6412358b50165: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:00:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.051018 13121 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:00:24.290 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:24.290 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:24.291 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2114), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [13ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [15ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [18ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [18ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [34ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [36ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [36ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [74ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [76ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [78ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [214ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [216ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [218ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [218ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [369ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [374ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [376ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [377ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [377ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2110ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [2114ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1999048121(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@32a187f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239) -> (continuation of Deferred@1919228625 after retry RPC@1006536539) -> (continuation of Deferred@2087664384 after retry RPC@650789379) -> (continuation of Deferred@820117111 after retry RPC@1154447410), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f75e964 -> wakeup thread Time-limited test -> (continuation of Deferred@75496263 after retry RPC@1912609368) -> (continuation of Deferred@377895930 after retry RPC@1206628239) -> (continuation of Deferred@1919228625 after retry RPC@1006536539) -> (continuation of Deferred@2087664384 after retry RPC@650789379) -> (continuation of Deferred@820117111 after retry RPC@1154447410))) after lookup
21:00:24.291 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:24.292 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] Successfully connected to peer
21:00:24.317 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe4a2c502, L:/127.0.0.1:58948 - R:/127.12.15.130:39631] using TOKEN/null
21:00:24.346 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.129:46601
21:00:24.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.348868 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.707290 13133 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.707609 13133 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.707705 13133 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.716153 13133 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.716321 13133 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.129
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.727187 13133 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.728957 13133 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:24.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.730887 13133 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.742374 13138 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.742576 13141 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:24.742735 13139 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.743309 13133 server_base.cc:1061] running on GCE node
21:00:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.744205 13133 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.746222 13133 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.747447 13133 hybrid_clock.cc:648] HybridClock initialized: now 1783544424747371 us; error 63 us; skew 500 ppm
21:00:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.747869 13133 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.750516 13133 webserver.cc:534] Webserver started at http://127.12.15.129:42207/ using document root <none> and password file <none>
21:00:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.751369 13133 fs_manager.cc:362] Metadata directory not provided
21:00:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.751482 13133 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:24.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.757511 13133 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
21:00:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.760872 13147 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.762511 13133 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.762735 13133 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.763497 13133 fs_report.cc:389] FS layout report
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/data
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.786418 13133 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.787412 13133 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.787688 13133 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.788647 13133 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.790985 13154 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:24.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.798103 13133 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:24.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.798209 13133 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.000s	sys 0.001s
21:00:24.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.798316 13133 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.801643 13133 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.801788 13133 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
21:00:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.802268 13154 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap starting.
21:00:24.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.826565 13154 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Log is configured to *not* fsync() on all Append() calls
21:00:24.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.841251 13154 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.842056 13154 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap complete.
21:00:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.843604 13154 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent bootstrapping tablet: real 0.042s	user 0.019s	sys 0.022s
21:00:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.850991 13154 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.852130 13154 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Initialized, Role: FOLLOWER
21:00:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.853096 13154 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.854842 13154 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent starting tablet: real 0.011s	user 0.009s	sys 0.002s
21:00:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.861013 13133 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.129:46601
21:00:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.861054 13262 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.129:46601 every 8 connection(s)
21:00:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.862475 13133 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:24.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.868669 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 13133
21:00:24.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.882521 13264 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:24.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.882965 13264 heartbeater.cc:461] Registering TS with master...
21:00:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.884155 13265 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.884177 13264 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.884375 13265 heartbeater.cc:461] Registering TS with master...
21:00:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.884385 13263 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.884572 13263 heartbeater.cc:461] Registering TS with master...
21:00:24.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.884963 13265 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:24.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.885212 13263 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:24.887 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:24.888 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.130:39631
21:00:24.888 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:00:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.887995 12533 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.889066 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 12768
21:00:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.889724 12463 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.889742 12533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.129:42623
21:00:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.891561 12392 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:24.902307 12890 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:25.000036 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 12768
21:00:25.018 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] handling channelInactive
21:00:25.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:25.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state READY due to: connection closed
21:00:25.019 [DEBUG - Time-limited test] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.020 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.021 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.029 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:25.029 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:25.030 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test)) after lookup
21:00:25.031 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.032 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.032 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.032 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.049 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:25.049 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:25.050 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [11ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [13ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [13ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873))) after lookup
21:00:25.050 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.051 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.051 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.051 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.071 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:25.071 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:25.072 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [11ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [13ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [13ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [31ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [32ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [32ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [51ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772))) after lookup
21:00:25.072 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.074 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.074 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.074 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.130 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:25.130 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:25.131 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=111), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [11ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [13ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [13ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [31ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [32ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [32ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [51ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [53ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [55ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [56ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [110ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384))) after lookup
21:00:25.132 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.133 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.133 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.133 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.429 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:25.430 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:25.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=411), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [11ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [13ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [13ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [31ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [32ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [32ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [51ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [53ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [55ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [56ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [110ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [112ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [114ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [114ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [406ms] refreshing cache from master, [406ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [410ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384) -> (continuation of Deferred@1984796242 after retry RPC@1293190444), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384) -> (continuation of Deferred@1984796242 after retry RPC@1293190444))) after lookup
21:00:25.431 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.433 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.433 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.433 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.869 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:25.869 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:25.870 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=850), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [11ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [13ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [13ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [31ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [32ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [32ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [51ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [53ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [55ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [56ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [110ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [112ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [114ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [114ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [406ms] refreshing cache from master, [406ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [410ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [412ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [414ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [414ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [846ms] refreshing cache from master, [846ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [849ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384) -> (continuation of Deferred@1984796242 after retry RPC@1293190444) -> (continuation of Deferred@1416242166 after retry RPC@167456482), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384) -> (continuation of Deferred@1984796242 after retry RPC@1293190444) -> (continuation of Deferred@1416242166 after retry RPC@167456482))) after lookup
21:00:25.871 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:25.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.872 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:25.893596 13263 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.036790 13277 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.037163 13277 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:26.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.039216 13277 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:26.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:26.041265 13150 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:26.043228 13150 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.047178 12984 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.048151 13150 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84
21:00:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.048797 13277 raft_consensus.cc:2749] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:00:26.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.349406 13282 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 5d6f60d583a5412c84b6412358b50165)
21:00:26.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.349643 13282 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:26.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.350437 13282 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:26.353276 12919 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.358880 13217 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" is_pre_election: true
21:00:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.359315 13217 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8152e70912024c1f84634d5e966b4c84 in term 1.
21:00:26.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.360284 12917 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165
21:00:26.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.360599 13282 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:00:26.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.360757 13282 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 5d6f60d583a5412c84b6412358b50165)
21:00:26.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.360862 13282 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 2 FOLLOWER]: Advancing to term 3
21:00:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.363106 13282 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.363790 13282 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 3 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.364348 13217 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.364560 13217 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 1 FOLLOWER]: Advancing to term 3
21:00:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:26.365365 12919 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.368461 13217 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8152e70912024c1f84634d5e966b4c84 in term 3.
21:00:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.368958 12917 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165
21:00:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.369338 13282 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Leader election won for term 3
21:00:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.369822 13282 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 LEADER]: Becoming Leader. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Running, Role: LEADER
21:00:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.370362 13282 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.374154 12533 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 reported cstate change: term changed from 2 to 3, leader changed from 5d6f60d583a5412c84b6412358b50165 (127.12.15.130) to 8152e70912024c1f84634d5e966b4c84 (127.12.15.131). New cstate: current_term: 3 leader_uuid: "8152e70912024c1f84634d5e966b4c84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: HEALTHY } } }
21:00:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:26.759022 12919 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 -> Peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Couldn't send request to peer 5d6f60d583a5412c84b6412358b50165. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.888820 13217 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Refusing update from remote peer 8152e70912024c1f84634d5e966b4c84: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
21:00:26.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:26.889912 13282 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
21:00:27.949 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:27.949 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:27.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2930), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [6ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [10ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [11ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [13ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [13ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [31ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [32ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [32ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [51ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [53ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [55ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [56ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [110ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [112ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [114ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [114ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [406ms] refreshing cache from master, [406ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [410ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [412ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [414ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [414ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [846ms] refreshing cache from master, [846ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [849ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [851ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [853ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [853ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2926ms] refreshing cache from master, [2926ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [2930ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@278972488(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@74653e86 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384) -> (continuation of Deferred@1984796242 after retry RPC@1293190444) -> (continuation of Deferred@1416242166 after retry RPC@167456482) -> (continuation of Deferred@689578611 after retry RPC@1635676506), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48dbd441 -> wakeup thread Time-limited test -> (continuation of Deferred@663550421 after retry RPC@836978873) -> (continuation of Deferred@1150962606 after retry RPC@1872192772) -> (continuation of Deferred@242734151 after retry RPC@1520189384) -> (continuation of Deferred@1984796242 after retry RPC@1293190444) -> (continuation of Deferred@1416242166 after retry RPC@167456482) -> (continuation of Deferred@689578611 after retry RPC@1635676506))) after lookup
21:00:27.951 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:27.952 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] Successfully connected to peer
21:00:27.969 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x45200446, L:/127.0.0.1:47208 - R:/127.12.15.131:42675] using TOKEN/null
21:00:27.998 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.130:39631
21:00:27.999 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.000046 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:39631
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42015
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.355981 13321 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.356351 13321 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.356454 13321 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.364764 13321 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.364938 13321 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.130
21:00:28.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.375568 13321 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:28.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:39631
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42015
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.377297 13321 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.379103 13321 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.388613 13331 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.388661 13328 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:28.388707 13329 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.389554 13321 server_base.cc:1061] running on GCE node
21:00:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.390174 13321 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:28.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.391250 13321 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:28.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.392509 13321 hybrid_clock.cc:648] HybridClock initialized: now 1783544428392439 us; error 53 us; skew 500 ppm
21:00:28.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.393028 13321 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.395412 13321 webserver.cc:534] Webserver started at http://127.12.15.130:42015/ using document root <none> and password file <none>
21:00:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.396245 13321 fs_manager.cc:362] Metadata directory not provided
21:00:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.396410 13321 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:28.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.401979 13321 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
21:00:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.405102 13337 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.406504 13321 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.001s
21:00:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.406692 13321 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.407910 13321 fs_report.cc:389] FS layout report
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/data
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:28.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.428860 13321 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:28.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.429847 13321 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:28.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.430202 13321 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:28.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.431327 13321 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:28.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.433921 13344 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.440461 13321 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.440594 13321 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
21:00:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.440737 13321 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:28.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.444139 13321 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:28.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.444285 13321 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
21:00:28.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.444711 13344 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap starting.
21:00:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.469022 13344 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Log is configured to *not* fsync() on all Append() calls
21:00:28.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.491652 13321 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.130:39631
21:00:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.491706 13451 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.130:39631 every 8 connection(s)
21:00:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.493506 13321 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.499111 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 13321
21:00:28.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.499580 13344 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.500609 13344 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap complete.
21:00:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.509903 13344 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent bootstrapping tablet: real 0.066s	user 0.032s	sys 0.024s
21:00:28.533 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:28.534 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.131:42675
21:00:28.534 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:00:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.535212 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 12902
21:00:28.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.522476 13344 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:28.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.543756 13344 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Initialized, Role: FOLLOWER
21:00:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.545071 13344 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:28.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.553382 13344 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent starting tablet: real 0.043s	user 0.010s	sys 0.003s
21:00:28.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.569654 13452 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:28.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.570111 13452 heartbeater.cc:461] Registering TS with master...
21:00:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.570874 13454 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.571029 13454 heartbeater.cc:461] Registering TS with master...
21:00:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.571729 13452 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.572248 13454 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.572286 13453 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.572542 13453 heartbeater.cc:461] Registering TS with master...
21:00:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.573137 13453 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:28.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.575270 12533 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:28.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.575357 12463 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.576897 12533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.130:39673
21:00:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.578236 12392 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.580030 13452 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.584343 13024 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:28.685101 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 12902
21:00:28.700 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] handling channelInactive
21:00:28.701 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state READY due to: connection closed
21:00:28.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:28.702 [DEBUG - Time-limited test] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:28.703 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.703 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.703 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.709 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:28.709 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:28.709 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test)) after lookup
21:00:28.710 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:28.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.711 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.729 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:28.729 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:28.729 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [9ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [10ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044))) after lookup
21:00:28.730 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:28.731 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.732 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.732 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.769 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:28.769 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:28.770 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [9ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [10ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [29ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [31ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725))) after lookup
21:00:28.771 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:28.772 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.772 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.772 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.829 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:28.829 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:28.829 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=128), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [9ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [10ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [29ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [31ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [69ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [71ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [127ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174))) after lookup
21:00:28.830 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:28.831 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.831 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.832 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.909 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:28.910 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:28.910 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=208), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [9ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [10ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [29ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [31ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [69ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [71ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [127ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [129ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [130ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [131ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [204ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [208ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174) -> (continuation of Deferred@1192467976 after retry RPC@1896805629), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174) -> (continuation of Deferred@1192467976 after retry RPC@1896805629))) after lookup
21:00:28.911 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:28.912 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.912 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:28.912 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:29.709 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:29.710 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:29.712 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1010), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [9ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [10ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [29ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [31ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [69ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [71ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [127ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [129ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [130ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [131ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [204ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [208ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [209ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [211ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [211ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1005ms] refreshing cache from master, [1005ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [1008ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174) -> (continuation of Deferred@1192467976 after retry RPC@1896805629) -> (continuation of Deferred@1641134371 after retry RPC@1744703736), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174) -> (continuation of Deferred@1192467976 after retry RPC@1896805629) -> (continuation of Deferred@1641134371 after retry RPC@1744703736))) after lookup
21:00:29.712 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:29.714 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:29.714 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:29.714 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.972414 13467 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 8152e70912024c1f84634d5e966b4c84)
21:00:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.972599 13467 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.973222 13467 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:29.977140 13150 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.982481 13407 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165" is_pre_election: true
21:00:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.983080 13407 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 2.
21:00:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.984318 13150 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 8152e70912024c1f84634d5e966b4c84
21:00:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.984722 13467 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Leader pre-election won for term 4
21:00:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.984848 13467 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 8152e70912024c1f84634d5e966b4c84)
21:00:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.984917 13467 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 3 FOLLOWER]: Advancing to term 4
21:00:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.987064 13467 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.987677 13467 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 4 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.988289 13407 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.988464 13407 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 2 FOLLOWER]: Advancing to term 4
21:00:29.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:29.989614 13150 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.992825 13407 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 4.
21:00:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.993355 13150 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 8152e70912024c1f84634d5e966b4c84
21:00:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.993726 13467 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 FOLLOWER]: Leader election won for term 4
21:00:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.994155 13467 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 LEADER]: Becoming Leader. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Running, Role: LEADER
21:00:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.994715 13467 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:29.998944 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 reported cstate change: term changed from 3 to 4, leader changed from 8152e70912024c1f84634d5e966b4c84 (127.12.15.131) to 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129). New cstate: current_term: 4 leader_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:30.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:30.427377 13150 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 -> Peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Couldn't send request to peer 8152e70912024c1f84634d5e966b4c84. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:30.453783 13407 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Refusing update from remote peer 83ca46c2c5f54616bd88be0a9bb0a7b6: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
21:00:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:30.454650 13467 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
21:00:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:31.422515 13150 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111) [suppressed 4 similar messages]
21:00:32.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:32.430 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:32.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3729), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [9ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [10ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [29ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [31ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [69ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [71ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [127ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [129ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [130ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [131ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [204ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [208ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [209ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [211ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [211ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1005ms] refreshing cache from master, [1005ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [1008ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [1011ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1013ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1013ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [3725ms] refreshing cache from master, [3725ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [3728ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@805876557(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@5a3cd9ee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174) -> (continuation of Deferred@1192467976 after retry RPC@1896805629) -> (continuation of Deferred@1641134371 after retry RPC@1744703736) -> (continuation of Deferred@1599553557 after retry RPC@1220141783), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10fe2e5f -> wakeup thread Time-limited test -> (continuation of Deferred@2128552678 after retry RPC@523397044) -> (continuation of Deferred@1471034859 after retry RPC@1538127725) -> (continuation of Deferred@188981716 after retry RPC@1063332174) -> (continuation of Deferred@1192467976 after retry RPC@1896805629) -> (continuation of Deferred@1641134371 after retry RPC@1744703736) -> (continuation of Deferred@1599553557 after retry RPC@1220141783))) after lookup
21:00:32.431 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:32.432 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] Successfully connected to peer
21:00:32.453 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x4598d44b, L:/127.0.0.1:33648 - R:/127.12.15.129:46601] using TOKEN/null
21:00:32.485 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.131:42675
21:00:32.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.486814 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:42675
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36255
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:32.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.850538 13505 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:32.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.850884 13505 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:32.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.850968 13505 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.859750 13505 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.859917 13505 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.131
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.870982 13505 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:42675
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36255
21:00:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:32.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.873234 13505 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:32.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.875494 13505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:32.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.885860 13511 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:32.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.886368 13510 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.886821 13513 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.887858 13505 server_base.cc:1061] running on GCE node
21:00:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:32.888283 13150 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 -> Peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Couldn't send request to peer 8152e70912024c1f84634d5e966b4c84. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:00:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.888698 13505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.889743 13505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.890939 13505 hybrid_clock.cc:648] HybridClock initialized: now 1783544432890873 us; error 47 us; skew 500 ppm
21:00:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.891260 13505 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.893676 13505 webserver.cc:534] Webserver started at http://127.12.15.131:36255/ using document root <none> and password file <none>
21:00:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.894383 13505 fs_manager.cc:362] Metadata directory not provided
21:00:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.894481 13505 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:32.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.899993 13505 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.002s
21:00:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.902863 13521 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.904502 13505 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
21:00:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.904681 13505 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:19 on dist-test-slave-9w0b"
21:00:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.905628 13505 fs_report.cc:389] FS layout report
21:00:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/data
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.928427 13505 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.929309 13505 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.929555 13505 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:32.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.930503 13505 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:32.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.932972 13528 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:32.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.940279 13505 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:32.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.940383 13505 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
21:00:32.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.940510 13505 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:32.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.943827 13505 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:32.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.943974 13505 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
21:00:32.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.944566 13528 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap starting.
21:00:32.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.971773 13528 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Log is configured to *not* fsync() on all Append() calls
21:00:32.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.997797 13528 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:32.998821 13528 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap complete.
21:00:33.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.000818 13528 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent bootstrapping tablet: real 0.057s	user 0.042s	sys 0.012s
21:00:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.002986 13505 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.131:42675
21:00:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.003041 13635 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.131:42675 every 8 connection(s)
21:00:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.004570 13505 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:33.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.007673 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 13505
21:00:33.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:33.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.014940 13528 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:33.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.016211 13528 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Initialized, Role: FOLLOWER
21:00:33.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.017148 13528 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.023396 13528 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent starting tablet: real 0.022s	user 0.013s	sys 0.000s
21:00:33.028 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:33.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.047050 13636 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:33.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.047555 13636 heartbeater.cc:461] Registering TS with master...
21:00:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.048961 13636 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:33.049 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.129:46601
21:00:33.049 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.049894 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 13133
21:00:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.052498 12534 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.053756 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.131:33095
21:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.058086 13636 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.072175 13637 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.072391 13637 heartbeater.cc:461] Registering TS with master...
21:00:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.072726 13637 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.074460 12392 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:33.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.080869 13258 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.088234 13638 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.088471 13638 heartbeater.cc:461] Registering TS with master...
21:00:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.088864 13638 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.090464 12463 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:33.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:33.182436 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 13133
21:00:33.201 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:33.201 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] handling channelInactive
21:00:33.201 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state READY due to: connection closed
21:00:33.202 [DEBUG - Time-limited test] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.203 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.203 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.203 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.208 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:33.209 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:33.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test)) after lookup
21:00:33.210 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.211 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.229 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:33.230 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:33.230 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [7ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [9ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [9ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860))) after lookup
21:00:33.231 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.233 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.249 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:33.249 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:33.250 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [7ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [9ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [9ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [31ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [31ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [47ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160))) after lookup
21:00:33.250 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.251 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.251 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.251 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.288 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:33.289 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:33.289 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [7ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [9ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [9ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [31ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [31ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [47ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [48ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882))) after lookup
21:00:33.290 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.291 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.291 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.291 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.569 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:33.570 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:33.570 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=368), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [7ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [9ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [9ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [31ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [31ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [47ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [48ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [87ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [89ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [89ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [364ms] refreshing cache from master, [364ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [367ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882) -> (continuation of Deferred@785198250 after retry RPC@1451263153), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882) -> (continuation of Deferred@785198250 after retry RPC@1451263153))) after lookup
21:00:33.571 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.571 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.572 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.572 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.849 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:33.849 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:33.850 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=648), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [7ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [9ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [9ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [31ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [31ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [47ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [48ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [87ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [89ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [89ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [364ms] refreshing cache from master, [364ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [367ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [368ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [370ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [370ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [644ms] refreshing cache from master, [644ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [647ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882) -> (continuation of Deferred@785198250 after retry RPC@1451263153) -> (continuation of Deferred@982670648 after retry RPC@1759900631), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882) -> (continuation of Deferred@785198250 after retry RPC@1451263153) -> (continuation of Deferred@982670648 after retry RPC@1759900631))) after lookup
21:00:33.850 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:33.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:33.852 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:34.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.364440 13651 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:34.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.364706 13651 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.366497 13651 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:34.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:34.367738 13522 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:34.370451 13522 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.374070 13407 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165" is_pre_election: true
21:00:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.375103 13524 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84; no voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.375566 13651 raft_consensus.cc:2749] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.534561 13656 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.534976 13656 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.536691 13656 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:34.538131 13338 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:34.540098 13338 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.545527 13591 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.545967 13591 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 3.
21:00:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.547120 13340 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.547576 13656 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Leader pre-election won for term 5
21:00:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.547714 13656 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:34.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.547940 13656 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 4 FOLLOWER]: Advancing to term 5
21:00:34.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.550232 13656 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:34.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.550964 13656 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 5 election: Requested vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:34.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.551661 13591 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:34.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.551918 13591 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 3 FOLLOWER]: Advancing to term 5
21:00:34.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:34.552587 13338 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:34.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.555961 13591 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 5.
21:00:34.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.556444 13340 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:34.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.556744 13656 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 FOLLOWER]: Leader election won for term 5
21:00:34.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.557207 13656 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 LEADER]: Becoming Leader. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Running, Role: LEADER
21:00:34.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.557782 13656 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.561877 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 reported cstate change: term changed from 4 to 5, leader changed from 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129) to 5d6f60d583a5412c84b6412358b50165 (127.12.15.130). New cstate: current_term: 5 leader_uuid: "5d6f60d583a5412c84b6412358b50165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:34.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:34.941807 13338 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 -> Peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Couldn't send request to peer 83ca46c2c5f54616bd88be0a9bb0a7b6. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:34.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.945791 13591 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Refusing update from remote peer 5d6f60d583a5412c84b6412358b50165: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
21:00:34.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:34.946729 13656 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
21:00:36.909 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:36.909 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:36.910 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=3708), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [6ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [7ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [9ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [9ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [31ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [31ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [47ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [48ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [87ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [89ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [89ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [364ms] refreshing cache from master, [364ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [367ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [368ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [370ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [370ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [644ms] refreshing cache from master, [644ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [647ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [648ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [650ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [650ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [3704ms] refreshing cache from master, [3704ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [3707ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@628545339(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@46800f4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882) -> (continuation of Deferred@785198250 after retry RPC@1451263153) -> (continuation of Deferred@982670648 after retry RPC@1759900631) -> (continuation of Deferred@774209111 after retry RPC@1771653903), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@472f8e10 -> wakeup thread Time-limited test -> (continuation of Deferred@242476860 after retry RPC@890746860) -> (continuation of Deferred@1363085204 after retry RPC@1495233160) -> (continuation of Deferred@1683914107 after retry RPC@1190647882) -> (continuation of Deferred@785198250 after retry RPC@1451263153) -> (continuation of Deferred@982670648 after retry RPC@1759900631) -> (continuation of Deferred@774209111 after retry RPC@1771653903))) after lookup
21:00:36.910 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:36.911 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] Successfully connected to peer
21:00:36.928 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xbeab88e8, L:/127.0.0.1:52302 - R:/127.12.15.130:39631] using TOKEN/null
21:00:36.954 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.129:46601
21:00:36.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:36.955714 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.297940 13693 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.298316 13693 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.298444 13693 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.310568 13693 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.310840 13693 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.129
21:00:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.322444 13693 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.324270 13693 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:37.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.326052 13693 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.336357 13699 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.336483 13702 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.336390 13700 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:37.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.337904 13693 server_base.cc:1061] running on GCE node
21:00:37.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.338442 13693 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:37.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.339414 13693 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:37.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.340637 13693 hybrid_clock.cc:648] HybridClock initialized: now 1783544437340544 us; error 82 us; skew 500 ppm
21:00:37.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.341068 13693 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.343531 13693 webserver.cc:534] Webserver started at http://127.12.15.129:42207/ using document root <none> and password file <none>
21:00:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.344264 13693 fs_manager.cc:362] Metadata directory not provided
21:00:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.344359 13693 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.351655 13693 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.000s
21:00:37.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.355178 13708 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.356702 13693 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
21:00:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.356902 13693 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.357769 13693 fs_report.cc:389] FS layout report
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/data
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.387274 13693 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.388164 13693 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.388415 13693 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:37.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.389416 13693 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:37.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.391875 13715 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.398048 13693 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.398185 13693 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
21:00:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.398314 13693 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.401750 13693 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.401865 13693 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
21:00:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.402366 13715 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap starting.
21:00:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:37.427444 13338 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 -> Peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Couldn't send request to peer 83ca46c2c5f54616bd88be0a9bb0a7b6. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:00:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.428390 13715 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Log is configured to *not* fsync() on all Append() calls
21:00:37.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.451753 13693 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.129:46601
21:00:37.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.451840 13822 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.129:46601 every 8 connection(s)
21:00:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.453235 13693 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.455227 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 13693
21:00:37.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:37.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.466125 13715 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.469973 13715 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap complete.
21:00:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.478427 13715 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent bootstrapping tablet: real 0.076s	user 0.039s	sys 0.019s
21:00:37.490 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.490144 13715 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.491103 13715 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Initialized, Role: FOLLOWER
21:00:37.491 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.130:39631
21:00:37.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.492146 13715 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:37.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:00:37.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.499433 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 13321
21:00:37.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.500885 13823 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:37.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.501288 13823 heartbeater.cc:461] Registering TS with master...
21:00:37.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.501713 13824 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:37.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.501902 13824 heartbeater.cc:461] Registering TS with master...
21:00:37.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.502535 13823 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:37.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.502715 13824 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.505767 12534 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:37.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.508055 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.129:52771
21:00:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.509637 12392 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.512200 13825 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.512394 13825 heartbeater.cc:461] Registering TS with master...
21:00:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.512863 13825 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.514832 12463 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.520474 13715 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent starting tablet: real 0.042s	user 0.013s	sys 0.003s
21:00:37.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.521407 13823 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.533317 13447 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:37.633143 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 13321
21:00:37.652 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] handling channelInactive
21:00:37.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state READY due to: connection closed
21:00:37.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:37.653 [DEBUG - Time-limited test] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:37.654 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.654 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.654 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.668 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:37.668 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:37.669 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test)) after lookup
21:00:37.670 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:37.671 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.671 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.671 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.689 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:37.690 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:37.690 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [16ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [18ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [18ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357))) after lookup
21:00:37.691 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:37.692 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.692 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.692 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.729 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:37.729 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:37.730 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [16ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [18ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [18ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [37ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [39ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [39ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [76ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460))) after lookup
21:00:37.730 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:37.732 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.732 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.732 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.815 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:37.815 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:37.816 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=163), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [16ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [18ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [18ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [37ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [39ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [39ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [76ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [77ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [79ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [162ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307))) after lookup
21:00:37.817 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:37.818 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.818 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.818 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.849 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:37.850 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:37.850 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=197), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [16ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [18ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [18ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [37ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [39ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [39ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [76ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [77ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [79ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [162ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [163ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [165ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [165ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307) -> (continuation of Deferred@1430916167 after retry RPC@596203606), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307) -> (continuation of Deferred@1430916167 after retry RPC@596203606))) after lookup
21:00:37.851 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:37.852 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.852 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:37.852 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:38.809 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:38.810 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:38.810 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1157), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [16ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [18ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [18ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [37ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [39ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [39ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [76ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [77ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [79ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [162ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [163ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [165ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [165ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [197ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [199ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [199ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1153ms] refreshing cache from master, [1153ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [1156ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307) -> (continuation of Deferred@1430916167 after retry RPC@596203606) -> (continuation of Deferred@1346024179 after retry RPC@957877066), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307) -> (continuation of Deferred@1430916167 after retry RPC@596203606) -> (continuation of Deferred@1346024179 after retry RPC@957877066))) after lookup
21:00:38.811 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:38.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:38.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:38.813 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.893378 13838 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 5d6f60d583a5412c84b6412358b50165)
21:00:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.893563 13838 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:38.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.894186 13838 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:38.897181 13524 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.903162 13778 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" is_pre_election: true
21:00:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.903728 13778 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8152e70912024c1f84634d5e966b4c84 in term 4.
21:00:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.904812 13522 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165
21:00:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.905143 13838 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Leader pre-election won for term 6
21:00:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.905296 13838 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 5d6f60d583a5412c84b6412358b50165)
21:00:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.905414 13838 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 5 FOLLOWER]: Advancing to term 6
21:00:38.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.907646 13838 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:38.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.908418 13838 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 6 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:38.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.909003 13778 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:38.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.909224 13778 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 4 FOLLOWER]: Advancing to term 6
21:00:38.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:38.910022 13524 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.913326 13778 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8152e70912024c1f84634d5e966b4c84 in term 6.
21:00:38.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.914012 13522 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165
21:00:38.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.914398 13838 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 FOLLOWER]: Leader election won for term 6
21:00:38.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.914948 13838 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 LEADER]: Becoming Leader. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Running, Role: LEADER
21:00:38.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.915475 13838 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:38.920195 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 reported cstate change: term changed from 5 to 6, leader changed from 5d6f60d583a5412c84b6412358b50165 (127.12.15.130) to 8152e70912024c1f84634d5e966b4c84 (127.12.15.131). New cstate: current_term: 6 leader_uuid: "8152e70912024c1f84634d5e966b4c84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: HEALTHY } } }
21:00:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:39.421631 13524 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111) [suppressed 2 similar messages]
21:00:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:39.423297 13524 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 -> Peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Couldn't send request to peer 5d6f60d583a5412c84b6412358b50165. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:39.437508 13778 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Refusing update from remote peer 8152e70912024c1f84634d5e966b4c84: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
21:00:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:39.438433 13856 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
21:00:41.149 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:41.150 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:41.150 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3497), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [15ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [16ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [18ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [18ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [37ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [39ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [39ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [76ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [77ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [79ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [162ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [163ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [165ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [165ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [197ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [199ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [199ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1153ms] refreshing cache from master, [1153ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [1156ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [1158ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [1160ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [1160ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [3493ms] refreshing cache from master, [3493ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [3496ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@877853395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6764a8bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307) -> (continuation of Deferred@1430916167 after retry RPC@596203606) -> (continuation of Deferred@1346024179 after retry RPC@957877066) -> (continuation of Deferred@1258775535 after retry RPC@842822558), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a778f -> wakeup thread Time-limited test -> (continuation of Deferred@956534109 after retry RPC@1786127357) -> (continuation of Deferred@1399636623 after retry RPC@140405460) -> (continuation of Deferred@1457040134 after retry RPC@757818307) -> (continuation of Deferred@1430916167 after retry RPC@596203606) -> (continuation of Deferred@1346024179 after retry RPC@957877066) -> (continuation of Deferred@1258775535 after retry RPC@842822558))) after lookup
21:00:41.151 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:41.152 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] Successfully connected to peer
21:00:41.168 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x82b87628, L:/127.0.0.1:50486 - R:/127.12.15.131:42675] using TOKEN/null
21:00:41.194 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.130:39631
21:00:41.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.197402 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:39631
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42015
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.577742 13879 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.578135 13879 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.578255 13879 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:41.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.587144 13879 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:41.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.587381 13879 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.130
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.598119 13879 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:39631
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42015
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.600314 13879 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:41.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.602314 13879 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.613317 13889 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.613354 13886 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:41.613235 13887 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:41.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.614001 13879 server_base.cc:1061] running on GCE node
21:00:41.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.614744 13879 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:41.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.615869 13879 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:41.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.617096 13879 hybrid_clock.cc:648] HybridClock initialized: now 1783544441617022 us; error 62 us; skew 500 ppm
21:00:41.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.617489 13879 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:41.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.619925 13879 webserver.cc:534] Webserver started at http://127.12.15.130:42015/ using document root <none> and password file <none>
21:00:41.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.620774 13879 fs_manager.cc:362] Metadata directory not provided
21:00:41.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.620922 13879 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:41.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.626652 13879 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
21:00:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.629700 13895 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:41.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.631255 13879 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.631464 13879 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.632254 13879 fs_report.cc:389] FS layout report
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/data
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.652941 13879 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:41.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.654016 13879 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:41.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.654408 13879 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.655658 13879 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.658098 13902 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:41.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.664105 13879 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:41.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.664235 13879 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
21:00:41.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.664372 13879 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.667554 13879 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.667800 13879 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.003s
21:00:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.668283 13902 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap starting.
21:00:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.696062 13902 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Log is configured to *not* fsync() on all Append() calls
21:00:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.720537 13879 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.130:39631
21:00:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.720574 14009 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.130:39631 every 8 connection(s)
21:00:41.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.722055 13879 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:41.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.725204 13902 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:41.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.725994 13902 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap complete.
21:00:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.727596 13902 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent bootstrapping tablet: real 0.060s	user 0.046s	sys 0.011s
21:00:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.729322 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 13879
21:00:41.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.736173 13902 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.737465 13902 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Initialized, Role: FOLLOWER
21:00:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.738318 13902 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:41.771 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:41.771 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.131:42675
21:00:41.772 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:00:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.772640 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 13505
21:00:41.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.773957 14010 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:41.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.774492 14010 heartbeater.cc:461] Registering TS with master...
21:00:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.775913 14010 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:41.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.777750 13902 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent starting tablet: real 0.050s	user 0.019s	sys 0.028s
21:00:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.779074 14011 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.779263 14011 heartbeater.cc:461] Registering TS with master...
21:00:41.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.779876 14011 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:41.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.780810 14012 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.780841 12533 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.780954 14012 heartbeater.cc:461] Registering TS with master...
21:00:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.781440 14012 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.782608 12533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.130:50281
21:00:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.782946 12462 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:41.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.783842 12392 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:41.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.785967 14010 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:41.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.800945 13631 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:41.905194 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 13505
21:00:41.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:41.923 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] handling channelInactive
21:00:41.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state READY due to: connection closed
21:00:41.924 [DEBUG - Time-limited test] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:41.925 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.925 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.925 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.949 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:41.950 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:41.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [25ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1045743554(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2d84b2c2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15132c9c -> wakeup thread Time-limited test)) after lookup
21:00:41.951 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:41.952 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.952 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.952 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.969 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:41.969 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:41.969 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [25ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [26ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [28ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [28ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [45ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1045743554(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2d84b2c2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15132c9c -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184))) after lookup
21:00:41.970 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:41.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.972 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.989 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:41.989 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:41.990 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [25ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [26ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [28ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [28ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [45ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [46ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [48ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [48ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [65ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1045743554(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2d84b2c2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15132c9c -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285))) after lookup
21:00:41.990 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:41.991 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.991 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:41.992 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:42.009 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:42.009 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:42.010 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=86), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [25ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [26ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [28ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [28ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [45ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [46ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [48ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [48ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [65ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [66ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [68ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [68ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [85ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1045743554(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2d84b2c2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285) -> (continuation of Deferred@2021041615 after retry RPC@204146751), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15132c9c -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285) -> (continuation of Deferred@2021041615 after retry RPC@204146751))) after lookup
21:00:42.011 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:42.012 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:42.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:42.012 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:42.529 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:42.530 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:42.530 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=606), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [25ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [26ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [28ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [28ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [45ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [46ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [48ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [48ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [65ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [66ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [68ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [68ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [85ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [86ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [88ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [88ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [602ms] refreshing cache from master, [602ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [605ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1045743554(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2d84b2c2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285) -> (continuation of Deferred@2021041615 after retry RPC@204146751) -> (continuation of Deferred@159265188 after retry RPC@1572911684), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15132c9c -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285) -> (continuation of Deferred@2021041615 after retry RPC@204146751) -> (continuation of Deferred@159265188 after retry RPC@1572911684))) after lookup
21:00:42.531 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:42.532 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:42.532 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:42.532 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:43.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.246675 14025 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 8152e70912024c1f84634d5e966b4c84)
21:00:43.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.246995 14025 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.248732 14025 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:43.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:43.250572 13711 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:43.256420 13711 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.257903 13965 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165" is_pre_election: true
21:00:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.258618 13965 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 5.
21:00:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.259806 13711 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 8152e70912024c1f84634d5e966b4c84
21:00:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.260288 14025 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Leader pre-election won for term 7
21:00:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.260444 14025 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 8152e70912024c1f84634d5e966b4c84)
21:00:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.260530 14025 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 6 FOLLOWER]: Advancing to term 7
21:00:43.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.269100 14025 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:43.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.269982 14025 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 7 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:43.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.270587 13965 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:43.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.270864 13965 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 5 FOLLOWER]: Advancing to term 7
21:00:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:43.272102 13711 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:43.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.276674 13965 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 7.
21:00:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.277241 13711 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 8152e70912024c1f84634d5e966b4c84
21:00:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.277594 14025 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Leader election won for term 7
21:00:43.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.278090 14025 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 LEADER]: Becoming Leader. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Running, Role: LEADER
21:00:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.279017 14025 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.283272 12533 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 reported cstate change: term changed from 6 to 7, leader changed from 8152e70912024c1f84634d5e966b4c84 (127.12.15.131) to 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129). New cstate: current_term: 7 leader_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.720456 13965 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Refusing update from remote peer 83ca46c2c5f54616bd88be0a9bb0a7b6: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
21:00:43.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:43.721618 14025 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
21:00:43.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:43.740141 13711 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 -> Peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Couldn't send request to peer 8152e70912024c1f84634d5e966b4c84. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:44.029 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:44.030 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:44.030 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2106), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [1ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [25ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [26ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [28ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [28ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [45ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [46ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [48ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [48ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [65ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [66ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [68ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [68ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [85ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [86ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [88ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [88ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [602ms] refreshing cache from master, [602ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [605ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [607ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [608ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [608ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2102ms] refreshing cache from master, [2102ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [2105ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@1045743554(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2d84b2c2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285) -> (continuation of Deferred@2021041615 after retry RPC@204146751) -> (continuation of Deferred@159265188 after retry RPC@1572911684) -> (continuation of Deferred@210449493 after retry RPC@1421983002), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15132c9c -> wakeup thread Time-limited test -> (continuation of Deferred@834562967 after retry RPC@1027987184) -> (continuation of Deferred@1926437273 after retry RPC@1639101285) -> (continuation of Deferred@2021041615 after retry RPC@204146751) -> (continuation of Deferred@159265188 after retry RPC@1572911684) -> (continuation of Deferred@210449493 after retry RPC@1421983002))) after lookup
21:00:44.031 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.032 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] Successfully connected to peer
21:00:44.053 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x61d04a6e, L:/127.0.0.1:42724 - R:/127.12.15.129:46601] using TOKEN/null
21:00:44.078 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.131:42675
21:00:44.079 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.080094 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:42675
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36255
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.421942 14053 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.422267 14053 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.422351 14053 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:44.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.430815 14053 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:44.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.430990 14053 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.131
21:00:44.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.441351 14053 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:44.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:42675
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36255
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.443197 14053 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.444991 14053 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.455691 14058 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:44.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.457558 14059 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:44.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.459335 14061 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:44.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.459739 14053 server_base.cc:1061] running on GCE node
21:00:44.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.460716 14053 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.462007 14053 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.463251 14053 hybrid_clock.cc:648] HybridClock initialized: now 1783544444463198 us; error 47 us; skew 500 ppm
21:00:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.463644 14053 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:44.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.466645 14053 webserver.cc:534] Webserver started at http://127.12.15.131:36255/ using document root <none> and password file <none>
21:00:44.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.467669 14053 fs_manager.cc:362] Metadata directory not provided
21:00:44.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.467814 14053 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:44.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.475028 14053 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
21:00:44.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.477979 14067 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.479549 14053 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.479708 14053 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:19 on dist-test-slave-9w0b"
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.480407 14053 fs_report.cc:389] FS layout report
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/data
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:44.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.494220 14053 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.495170 14053 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.495414 14053 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.496320 14053 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.498440 14074 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.504913 14053 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.505028 14053 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
21:00:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.505136 14053 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.508462 14053 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.508565 14053 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.002s
21:00:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.509189 14074 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap starting.
21:00:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.538298 14074 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Log is configured to *not* fsync() on all Append() calls
21:00:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.564751 14053 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.131:42675
21:00:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.564788 14181 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.131:42675 every 8 connection(s)
21:00:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.566253 14053 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:44.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.571408 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 14053
21:00:44.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.584944 14074 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:44.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.585901 14074 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap complete.
21:00:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.594914 14074 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent bootstrapping tablet: real 0.084s	user 0.049s	sys 0.019s
21:00:44.598 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:44.599 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.129:46601
21:00:44.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:44.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.600529 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 13693
21:00:44.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.606925 14074 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:44.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.608299 14074 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Initialized, Role: FOLLOWER
21:00:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.609426 14074 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.616325 14182 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.616771 14182 heartbeater.cc:461] Registering TS with master...
21:00:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.618033 14182 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:44.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.621992 12533 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.623953 14184 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.624110 14184 heartbeater.cc:461] Registering TS with master...
21:00:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.624691 14184 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.625211 12533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.131:42069
21:00:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.625622 14183 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.625787 14183 heartbeater.cc:461] Registering TS with master...
21:00:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.626413 14183 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.626547 12462 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:44.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.628105 12392 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:44.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.634125 14182 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:44.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.634100 14074 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent starting tablet: real 0.039s	user 0.017s	sys 0.016s
21:00:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.643817 13818 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:44.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:44.748000 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 13693
21:00:44.767 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] handling channelInactive
21:00:44.767 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:44.767 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state READY due to: connection closed
21:00:44.768 [DEBUG - Time-limited test] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:44.768360 14198 negotiation.cc:336] Failed RPC negotiation. Trace:
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.746534 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.12.15.129:60929 (local address 127.12.15.131:42675)
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749043 (+  2509us) server_negotiation.cc:207] Beginning negotiation
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749055 (+    12us) server_negotiation.cc:400] Waiting for connection header
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749100 (+    45us) server_negotiation.cc:408] Connection header received
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749234 (+   134us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749246 (+    12us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749568 (+   322us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.749808 (+   240us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:44.766960 (+ 17152us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.12.15.129:60929: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2253,"thread_start_us":1478,"threads_started":1}
21:00:44.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.769 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.789 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:44.789 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:44.790 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test)) after lookup
21:00:44.790 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.791 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.791 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.792 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.813 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:44.814 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:44.814 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [23ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [24ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [42ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686))) after lookup
21:00:44.814 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.815 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.816 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.816 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.849 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:44.850 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:44.850 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [23ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [24ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [42ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [47ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [82ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100))) after lookup
21:00:44.850 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.851 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.852 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.852 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.949 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:44.949 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:44.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=182), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [23ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [24ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [42ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [47ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [82ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [83ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [85ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [85ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [182ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109))) after lookup
21:00:44.950 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.952 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.952 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.952 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.989 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:44.989 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:44.989 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=221), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [23ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [24ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [42ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [47ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [82ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [83ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [85ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [85ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [182ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [183ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [185ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [185ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [218ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [221ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109) -> (continuation of Deferred@1835261171 after retry RPC@1573889665), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109) -> (continuation of Deferred@1835261171 after retry RPC@1573889665))) after lookup
21:00:44.990 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:44.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.991 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:44.991 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:45.288 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:45.289 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:45.289 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=521), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [23ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [24ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [42ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [47ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [82ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [83ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [85ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [85ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [182ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [183ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [185ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [185ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [218ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [221ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [222ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [224ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [224ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [518ms] refreshing cache from master, [518ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [521ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109) -> (continuation of Deferred@1835261171 after retry RPC@1573889665) -> (continuation of Deferred@572667784 after retry RPC@49899235), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109) -> (continuation of Deferred@1835261171 after retry RPC@1573889665) -> (continuation of Deferred@572667784 after retry RPC@49899235))) after lookup
21:00:45.290 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:45.291 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:45.291 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:45.291 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.029251 14199 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.029452 14199 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.030987 14199 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:46.032159 13896 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:46.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:46.034322 13896 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.040350 14136 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.040926 14136 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 6.
21:00:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.042192 13898 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.042657 14199 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Leader pre-election won for term 8
21:00:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.042815 14199 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.042912 14199 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 7 FOLLOWER]: Advancing to term 8
21:00:46.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.044997 14199 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:46.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.045776 14199 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 8 election: Requested vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:46.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.046454 14136 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:46.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.046686 14136 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 6 FOLLOWER]: Advancing to term 8
21:00:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:46.047248 13896 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.051025 14136 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 8.
21:00:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.051651 13898 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:46.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.052143 14199 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 FOLLOWER]: Leader election won for term 8
21:00:46.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.052629 14199 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 LEADER]: Becoming Leader. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Running, Role: LEADER
21:00:46.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.053260 14199 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.057572 12533 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 reported cstate change: term changed from 7 to 8, leader changed from 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129) to 5d6f60d583a5412c84b6412358b50165 (127.12.15.130). New cstate: current_term: 8 leader_uuid: "5d6f60d583a5412c84b6412358b50165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.569154 14136 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Refusing update from remote peer 5d6f60d583a5412c84b6412358b50165: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
21:00:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:46.570199 14216 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
21:00:46.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:46.573607 13896 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 -> Peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Couldn't send request to peer 83ca46c2c5f54616bd88be0a9bb0a7b6. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:47.309 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:47.309 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:47.310 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2542), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [2ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [22ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [23ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [24ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [42ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [47ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [49ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [49ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [82ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [83ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [85ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [85ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [182ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [183ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [185ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [185ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [218ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [221ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [222ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [224ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [224ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [518ms] refreshing cache from master, [518ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [521ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [522ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [524ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [524ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2538ms] refreshing cache from master, [2538ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [2541ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@188099509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@20f13a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109) -> (continuation of Deferred@1835261171 after retry RPC@1573889665) -> (continuation of Deferred@572667784 after retry RPC@49899235) -> (continuation of Deferred@807580338 after retry RPC@2032469035), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3b021572 -> wakeup thread Time-limited test -> (continuation of Deferred@1747051333 after retry RPC@1134805686) -> (continuation of Deferred@1638803933 after retry RPC@2068921100) -> (continuation of Deferred@1753128045 after retry RPC@444941109) -> (continuation of Deferred@1835261171 after retry RPC@1573889665) -> (continuation of Deferred@572667784 after retry RPC@49899235) -> (continuation of Deferred@807580338 after retry RPC@2032469035))) after lookup
21:00:47.310 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:47.311 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] Successfully connected to peer
21:00:47.329 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc0bcfcf4, L:/127.0.0.1:45100 - R:/127.12.15.130:39631] using TOKEN/null
21:00:47.352 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.129:46601
21:00:47.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.353833 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:47.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.714285 14235 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:47.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.714666 14235 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.714809 14235 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.724207 14235 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.724483 14235 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.129
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.735661 14235 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.737982 14235 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:47.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.740190 14235 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.752395 14240 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.752530 14243 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:47.753196 14241 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.753577 14235 server_base.cc:1061] running on GCE node
21:00:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.754235 14235 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:47.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.755559 14235 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:47.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.756842 14235 hybrid_clock.cc:648] HybridClock initialized: now 1783544447756762 us; error 58 us; skew 500 ppm
21:00:47.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.757200 14235 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:47.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.759836 14235 webserver.cc:534] Webserver started at http://127.12.15.129:42207/ using document root <none> and password file <none>
21:00:47.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.760579 14235 fs_manager.cc:362] Metadata directory not provided
21:00:47.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.760674 14235 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.766242 14235 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
21:00:47.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.769362 14249 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.771234 14235 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.001s
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.771406 14235 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.772117 14235 fs_report.cc:389] FS layout report
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/data
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:47.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:47.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.792979 14235 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:47.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.793931 14235 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:47.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.794193 14235 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:47.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.795356 14235 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.797612 14256 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.804216 14235 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.804334 14235 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
21:00:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.804447 14235 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:47.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.807781 14235 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:47.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.807941 14235 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
21:00:47.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.808506 14256 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap starting.
21:00:47.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.836344 14256 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Log is configured to *not* fsync() on all Append() calls
21:00:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.865270 14235 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.129:46601
21:00:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.865332 14363 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.129:46601 every 8 connection(s)
21:00:47.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.866887 14235 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.876478 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 14235
21:00:47.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.882726 14256 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.883517 14256 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap complete.
21:00:47.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.885493 14256 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent bootstrapping tablet: real 0.078s	user 0.050s	sys 0.015s
21:00:47.893 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:47.893 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.130:39631
21:00:47.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:00:47.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.894517 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 13879
21:00:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.895435 14256 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.896839 14256 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Initialized, Role: FOLLOWER
21:00:47.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.897804 14256 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:47.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.927263 14005 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:47.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.930999 14366 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:47.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.931411 14366 heartbeater.cc:461] Registering TS with master...
21:00:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.933547 14366 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.935542 14365 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.935762 14365 heartbeater.cc:461] Registering TS with master...
21:00:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.936414 14365 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:47.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.937458 14364 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:47.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.937573 14364 heartbeater.cc:461] Registering TS with master...
21:00:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.938026 12462 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.938162 14364 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.938993 12391 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:47.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.940292 12533 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:47.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.941689 14256 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent starting tablet: real 0.056s	user 0.013s	sys 0.040s
21:00:47.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.941839 12533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.129:49341
21:00:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:47.944973 14364 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:48.041937 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 13879
21:00:48.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:48.060 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] handling channelInactive
21:00:48.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state READY due to: connection closed
21:00:48.061 [DEBUG - Time-limited test] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.063 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.063 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.063 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.069 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:48.069 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:48.069 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test)) after lookup
21:00:48.070 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.071 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.089 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:48.089 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:48.089 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [8ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [10ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265))) after lookup
21:00:48.090 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.091 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.091 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.091 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.129 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:48.129 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:48.129 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [8ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [10ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [30ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [30ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990))) after lookup
21:00:48.130 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.131 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:48.209 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:48.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=148), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [8ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [10ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [30ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [30ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [68ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [70ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [70ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [144ms] refreshing cache from master, [145ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [147ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379))) after lookup
21:00:48.210 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.211 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.508 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:48.508 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:48.509 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=448), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [8ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [10ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [30ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [30ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [68ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [70ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [70ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [144ms] refreshing cache from master, [145ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [147ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [149ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [150ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [150ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [444ms] refreshing cache from master, [444ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [447ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379) -> (continuation of Deferred@524583613 after retry RPC@9035160), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379) -> (continuation of Deferred@524583613 after retry RPC@9035160))) after lookup
21:00:48.509 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.510 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.511 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.511 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.649 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:48.649 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:48.649 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=588), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [8ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [10ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [30ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [30ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [68ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [70ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [70ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [144ms] refreshing cache from master, [145ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [147ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [149ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [150ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [150ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [444ms] refreshing cache from master, [444ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [447ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [448ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [450ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [450ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [585ms] refreshing cache from master, [585ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [587ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379) -> (continuation of Deferred@524583613 after retry RPC@9035160) -> (continuation of Deferred@2025979691 after retry RPC@912951478), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379) -> (continuation of Deferred@524583613 after retry RPC@9035160) -> (continuation of Deferred@2025979691 after retry RPC@912951478))) after lookup
21:00:48.650 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:48.651 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.651 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:48.651 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631
21:00:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.270987 14379 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.271304 14379 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.273470 14379 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:49.275866 14252 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:49.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:49.277951 14252 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:49.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.285717 14136 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.286955 14252 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84
21:00:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.287525 14379 raft_consensus.cc:2749] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
21:00:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.414835 14384 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 5d6f60d583a5412c84b6412358b50165)
21:00:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.415122 14384 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:49.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.417136 14384 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:49.419023 14070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:49.420961 14070 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.426502 14319 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" is_pre_election: true
21:00:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.426956 14319 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8152e70912024c1f84634d5e966b4c84 in term 7.
21:00:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.427919 14068 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165
21:00:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.428323 14384 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Leader pre-election won for term 9
21:00:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.428467 14384 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 5d6f60d583a5412c84b6412358b50165)
21:00:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.428568 14384 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 8 FOLLOWER]: Advancing to term 9
21:00:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.430650 14384 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.431381 14384 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 9 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.432050 14319 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "8152e70912024c1f84634d5e966b4c84" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.432260 14319 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 7 FOLLOWER]: Advancing to term 9
21:00:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:49.432919 14070 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111)
21:00:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.436431 14319 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8152e70912024c1f84634d5e966b4c84 in term 9.
21:00:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.437078 14068 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8152e70912024c1f84634d5e966b4c84, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 5d6f60d583a5412c84b6412358b50165
21:00:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.437420 14384 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 FOLLOWER]: Leader election won for term 9
21:00:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.437927 14384 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 LEADER]: Becoming Leader. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Running, Role: LEADER
21:00:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.438426 14384 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:49.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.442591 12533 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 reported cstate change: term changed from 8 to 9, leader changed from 5d6f60d583a5412c84b6412358b50165 (127.12.15.130) to 8152e70912024c1f84634d5e966b4c84 (127.12.15.131). New cstate: current_term: 9 leader_uuid: "8152e70912024c1f84634d5e966b4c84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: HEALTHY } } }
21:00:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.907049 14319 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Refusing update from remote peer 8152e70912024c1f84634d5e966b4c84: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
21:00:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:49.908015 14384 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
21:00:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:49.921245 14070 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 -> Peer 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631): Couldn't send request to peer 5d6f60d583a5412c84b6412358b50165. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.130:39631: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:51.349 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:51.349 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:51.349 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=3288), Traces: [0ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [2ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [7ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [8ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [10ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [27ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [28ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [30ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [30ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [67ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [68ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [70ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [70ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [144ms] refreshing cache from master, [145ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [147ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [149ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [150ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [150ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [444ms] refreshing cache from master, [444ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [447ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [448ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [450ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [450ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [585ms] refreshing cache from master, [585ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [587ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [589ms] sending RPC to server 5d6f60d583a5412c84b6412358b50165, [590ms] received response from server 5d6f60d583a5412c84b6412358b50165: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [590ms] delaying RPC due to: Network error: Failed to connect to peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631): Connection refused: /127.12.15.130:39631, [3284ms] refreshing cache from master, [3285ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [3287ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@270162110(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@6adfed4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379) -> (continuation of Deferred@524583613 after retry RPC@9035160) -> (continuation of Deferred@2025979691 after retry RPC@912951478) -> (continuation of Deferred@1567878775 after retry RPC@1069474391), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@218dcfcf -> wakeup thread Time-limited test -> (continuation of Deferred@1177700287 after retry RPC@151915265) -> (continuation of Deferred@1959320405 after retry RPC@224886990) -> (continuation of Deferred@774931115 after retry RPC@534775379) -> (continuation of Deferred@524583613 after retry RPC@9035160) -> (continuation of Deferred@2025979691 after retry RPC@912951478) -> (continuation of Deferred@1567878775 after retry RPC@1069474391))) after lookup
21:00:51.350 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:51.351 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] Successfully connected to peer
21:00:51.366 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x22e7b523, L:/127.0.0.1:52248 - R:/127.12.15.131:42675] using TOKEN/null
21:00:51.389 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.130:39631
21:00:51.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.390473 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:39631
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42015
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.739185 14418 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.739506 14418 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.739596 14418 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.748077 14418 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.748240 14418 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.130
21:00:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.758472 14418 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.130:39631
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.130
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42015
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.130
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/logs
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.760084 14418 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:51.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.761806 14418 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.772329 14424 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.772476 14423 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:51.772684 14426 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.773123 14418 server_base.cc:1061] running on GCE node
21:00:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.773794 14418 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:51.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.774957 14418 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.776170 14418 hybrid_clock.cc:648] HybridClock initialized: now 1783544451776102 us; error 53 us; skew 500 ppm
21:00:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.776597 14418 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.779026 14418 webserver.cc:534] Webserver started at http://127.12.15.130:42015/ using document root <none> and password file <none>
21:00:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.779721 14418 fs_manager.cc:362] Metadata directory not provided
21:00:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.779824 14418 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.785183 14418 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.000s	sys 0.005s
21:00:51.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.788141 14432 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:51.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.789458 14418 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.789592 14418 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.790234 14418 fs_report.cc:389] FS layout report
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/wal
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/data
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:51.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.804049 14418 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:51.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.804949 14418 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:51.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.805199 14418 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.806265 14418 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.808705 14439 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.816635 14418 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.816736 14418 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
21:00:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.816896 14418 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.820017 14418 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.820140 14418 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.002s
21:00:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.820690 14439 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap starting.
21:00:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.844848 14439 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Log is configured to *not* fsync() on all Append() calls
21:00:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.882939 14439 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.884079 14439 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Bootstrap complete.
21:00:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.885651 14439 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent bootstrapping tablet: real 0.065s	user 0.049s	sys 0.013s
21:00:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.890529 14418 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.130:39631
21:00:51.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.890861 14546 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.130:39631 every 8 connection(s)
21:00:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.892334 14418 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-1/data/info.pb
21:00:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.892189 14439 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.892891 14439 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Initialized, Role: FOLLOWER
21:00:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.893980 14439 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:51.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.898904 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 14418
21:00:51.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:51.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.903769 14439 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165: Time spent starting tablet: real 0.018s	user 0.006s	sys 0.011s
21:00:51.917 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:51.918 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.131:42675
21:00:51.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:00:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.919232 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 14053
21:00:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.951520 14548 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.952070 14548 heartbeater.cc:461] Registering TS with master...
21:00:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.953464 14548 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.955175 14549 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.955339 14549 heartbeater.cc:461] Registering TS with master...
21:00:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.955958 14549 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.957947 12392 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.958022 14547 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.958144 14547 heartbeater.cc:461] Registering TS with master...
21:00:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.958602 14547 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:51.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.959614 12462 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:51.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.960453 12534 ts_manager.cc:194] Re-registered known tserver with Master: 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631)
21:00:51.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.961709 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.130:45065
21:00:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.963037 14177 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:51.964213 14547 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:52.060616 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 14053
21:00:52.078 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] handling channelInactive
21:00:52.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:52.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state READY due to: connection closed
21:00:52.079 [DEBUG - Time-limited test] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.081 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.088 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:52.088 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:52.089 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test)) after lookup
21:00:52.089 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.090 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.090 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.090 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.108 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:52.109 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:52.109 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [11ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [11ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [26ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239))) after lookup
21:00:52.109 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.110 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.110 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.111 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.148 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:52.149 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:52.149 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [11ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [11ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [26ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [30ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [32ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [69ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189))) after lookup
21:00:52.149 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.150 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.150 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.151 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.289 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:52.289 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:52.289 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=210), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [11ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [11ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [26ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [30ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [32ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [69ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [70ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [209ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066))) after lookup
21:00:52.289 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.290 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.291 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.291 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.568 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:52.569 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:52.569 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=490), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [11ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [11ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [26ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [30ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [32ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [69ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [70ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [209ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [210ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [212ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [212ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [489ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066) -> (continuation of Deferred@173768349 after retry RPC@582066739), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066) -> (continuation of Deferred@173768349 after retry RPC@582066739))) after lookup
21:00:52.570 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.571 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.788 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:52.788 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)[L],83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:52.788 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=709), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [11ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [11ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [26ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [30ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [32ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [69ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [70ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [209ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [210ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [212ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [212ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [489ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [490ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [492ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [492ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [706ms] refreshing cache from master, [706ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [708ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066) -> (continuation of Deferred@173768349 after retry RPC@582066739) -> (continuation of Deferred@1638052948 after retry RPC@1311584873), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066) -> (continuation of Deferred@173768349 after retry RPC@582066739) -> (continuation of Deferred@1638052948 after retry RPC@1311584873))) after lookup
21:00:52.789 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] connecting to peer
21:00:52.790 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.790 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:52.790 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675
21:00:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.449394 14563 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 8152e70912024c1f84634d5e966b4c84)
21:00:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.449589 14563 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.450273 14563 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:53.453290 14252 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.458978 14502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165" is_pre_election: true
21:00:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.459515 14502 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 8.
21:00:53.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.460392 14252 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 8152e70912024c1f84634d5e966b4c84
21:00:53.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.460713 14563 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Leader pre-election won for term 10
21:00:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.460856 14563 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 8152e70912024c1f84634d5e966b4c84)
21:00:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.460960 14563 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 9 FOLLOWER]: Advancing to term 10
21:00:53.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.463092 14563 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:53.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.463778 14563 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 10 election: Requested vote from peers 5d6f60d583a5412c84b6412358b50165 (127.12.15.130:39631), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:53.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.464427 14502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "5d6f60d583a5412c84b6412358b50165"
21:00:53.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.464648 14502 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 8 FOLLOWER]: Advancing to term 10
21:00:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:53.465713 14252 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111)
21:00:53.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.468756 14502 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 83ca46c2c5f54616bd88be0a9bb0a7b6 in term 10.
21:00:53.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.469370 14252 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 83ca46c2c5f54616bd88be0a9bb0a7b6; no voters: 8152e70912024c1f84634d5e966b4c84
21:00:53.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.469729 14563 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 10 FOLLOWER]: Leader election won for term 10
21:00:53.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.470147 14563 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 10 LEADER]: Becoming Leader. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Running, Role: LEADER
21:00:53.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.470703 14563 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:53.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.475087 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 reported cstate change: term changed from 9 to 10, leader changed from 8152e70912024c1f84634d5e966b4c84 (127.12.15.131) to 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129). New cstate: current_term: 10 leader_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:53.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.958280 14502 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Refusing update from remote peer 83ca46c2c5f54616bd88be0a9bb0a7b6: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
21:00:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:53.959277 14563 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
21:00:53.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:53.979782 14252 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 -> Peer 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675): Couldn't send request to peer 8152e70912024c1f84634d5e966b4c84. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:54.517072 14252 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.131:42675: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:00:55.889 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:55.889 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:55.889 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3810), Traces: [0ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [2ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [11ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [11ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [26ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [30ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [32ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [69ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [70ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [71ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [209ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [210ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [212ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [212ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [489ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [490ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [492ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [492ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [706ms] refreshing cache from master, [706ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [708ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [710ms] sending RPC to server 8152e70912024c1f84634d5e966b4c84, [711ms] received response from server 8152e70912024c1f84634d5e966b4c84: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [711ms] delaying RPC due to: Network error: Failed to connect to peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675): Connection refused: /127.12.15.131:42675, [3806ms] refreshing cache from master, [3806ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [3809ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@587905030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@7323fbef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066) -> (continuation of Deferred@173768349 after retry RPC@582066739) -> (continuation of Deferred@1638052948 after retry RPC@1311584873) -> (continuation of Deferred@1109786186 after retry RPC@1698443805), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@739d25f3 -> wakeup thread Time-limited test -> (continuation of Deferred@105034449 after retry RPC@476753239) -> (continuation of Deferred@462238462 after retry RPC@1388586189) -> (continuation of Deferred@296281038 after retry RPC@1647923066) -> (continuation of Deferred@173768349 after retry RPC@582066739) -> (continuation of Deferred@1638052948 after retry RPC@1311584873) -> (continuation of Deferred@1109786186 after retry RPC@1698443805))) after lookup
21:00:55.890 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:55.891 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] Successfully connected to peer
21:00:55.908 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2fae3935, L:/127.0.0.1:53988 - R:/127.12.15.129:46601] using TOKEN/null
21:00:55.936 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.131:42675
21:00:55.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:55.937079 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:42675
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36255
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.283546 14603 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.283865 14603 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.283960 14603 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.292434 14603 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.292621 14603 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.131
21:00:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.303275 14603 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.131:42675
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.131
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36255
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.131
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/logs
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.305058 14603 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.307001 14603 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.318156 14610 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.318190 14613 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.318604 14611 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.319479 14603 server_base.cc:1061] running on GCE node
21:00:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.320228 14603 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.321341 14603 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.322603 14603 hybrid_clock.cc:648] HybridClock initialized: now 1783544456322555 us; error 34 us; skew 500 ppm
21:00:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.323107 14603 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:56.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.325672 14603 webserver.cc:534] Webserver started at http://127.12.15.131:36255/ using document root <none> and password file <none>
21:00:56.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.326586 14603 fs_manager.cc:362] Metadata directory not provided
21:00:56.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.326749 14603 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.333298 14603 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.001s
21:00:56.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.336799 14619 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:56.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.338487 14603 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
21:00:56.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.338665 14603 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:56.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:56.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:19 on dist-test-slave-9w0b"
21:00:56.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.339625 14603 fs_report.cc:389] FS layout report
21:00:56.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/wal
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/data
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.363564 14603 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:56.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.364581 14603 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:56.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.364836 14603 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:56.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.365968 14603 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.368420 14626 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.375298 14603 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.375398 14603 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
21:00:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.375510 14603 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.378945 14603 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.379098 14603 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.004s	sys 0.000s
21:00:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.379633 14626 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap starting.
21:00:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.409396 14626 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Log is configured to *not* fsync() on all Append() calls
21:00:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.419309 14603 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.131:42675
21:00:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.419399 14733 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.131:42675 every 8 connection(s)
21:00:56.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.420792 14603 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-2/data/info.pb
21:00:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.426420 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 14603
21:00:56.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.444605 14735 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.445066 14735 heartbeater.cc:461] Registering TS with master...
21:00:56.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.446205 14735 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.449602 12392 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:56.452 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:56.452 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.15.129:46601
21:00:56.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.453506 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 14235
21:00:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.462270 14734 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.462438 14734 heartbeater.cc:461] Registering TS with master...
21:00:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.463168 14734 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.468065 12534 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.470067 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.131:44955
21:00:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.490952 14736 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.491146 14736 heartbeater.cc:461] Registering TS with master...
21:00:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.491601 14736 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:56.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.493341 12462 ts_manager.cc:194] Re-registered known tserver with Master: 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:56.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.494232 14626 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.495043 14626 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Bootstrap complete.
21:00:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.495215 14359 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.496628 14626 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent bootstrapping tablet: real 0.118s	user 0.077s	sys 0.004s
21:00:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.503170 14626 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.504280 14626 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8152e70912024c1f84634d5e966b4c84, State: Initialized, Role: FOLLOWER
21:00:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.505167 14626 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:56.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.506569 14734 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:56.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.507696 14626 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84: Time spent starting tablet: real 0.011s	user 0.011s	sys 0.001s
21:00:56.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:56.603415 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 14235
21:00:56.619 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] handling channelInactive
21:00:56.619 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:56.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state READY due to: connection closed
21:00:56.620 [DEBUG - Time-limited test] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:56.620795 14749 negotiation.cc:336] Failed RPC negotiation. Trace:
21:00:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:56.602778 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.12.15.129:45335 (local address 127.12.15.131:42675)
21:00:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:56.603767 (+   989us) server_negotiation.cc:207] Beginning negotiation
21:00:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:56.603778 (+    11us) server_negotiation.cc:400] Waiting for connection header
21:00:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:56.603813 (+    35us) server_negotiation.cc:408] Connection header received
21:00:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:00:56.619172 (+ 15359us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.12.15.129:45335: BlockingRecv error: recv got EOF from 127.12.15.129:45335 (error 108)
21:00:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":840,"thread_start_us":710,"threads_started":1}
21:00:56.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.621 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.629 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:56.629 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:56.630 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@400158808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2b57829 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29b25a78 -> wakeup thread Time-limited test)) after lookup
21:00:56.630 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:56.632 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.632 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.632 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.652 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:56.652 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:56.652 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [12ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [12ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@400158808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2b57829 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29b25a78 -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926))) after lookup
21:00:56.653 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:56.654 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.654 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.654 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.668 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:56.668 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:56.668 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [12ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [12ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [33ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [34ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [34ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@400158808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2b57829 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29b25a78 -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878))) after lookup
21:00:56.669 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:56.670 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.670 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.670 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.728 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:56.728 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:56.728 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [12ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [12ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [33ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [34ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [34ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [49ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [50ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [50ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@400158808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2b57829 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878) -> (continuation of Deferred@1756947225 after retry RPC@2023520490), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29b25a78 -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878) -> (continuation of Deferred@1756947225 after retry RPC@2023520490))) after lookup
21:00:56.729 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:56.730 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.730 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:56.730 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:57.108 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:57.108 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631),8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)[L]]}]
21:00:57.109 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=489), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [12ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [12ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [33ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [34ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [34ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [49ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [50ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [50ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [109ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [110ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [110ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [488ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@400158808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2b57829 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878) -> (continuation of Deferred@1756947225 after retry RPC@2023520490) -> (continuation of Deferred@1819189105 after retry RPC@1999561022), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29b25a78 -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878) -> (continuation of Deferred@1756947225 after retry RPC@2023520490) -> (continuation of Deferred@1819189105 after retry RPC@1999561022))) after lookup
21:00:57.109 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] connecting to peer
21:00:57.110 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:57.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] cleaning up while in state CONNECTING due to: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:57.111 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601) for tablet d1bb5190405140dfa1141ac6f4c37eb3: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601
21:00:57.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.858956 14750 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:57.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.859169 14750 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:57.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.860774 14750 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:57.861827 14433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:57.863524 14433 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.869515 14688 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84" is_pre_election: true
21:00:57.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.869992 14688 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 9.
21:00:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.871089 14435 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.871537 14750 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Leader pre-election won for term 11
21:00:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.871644 14750 raft_consensus.cc:493] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 83ca46c2c5f54616bd88be0a9bb0a7b6)
21:00:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.871721 14750 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 10 FOLLOWER]: Advancing to term 11
21:00:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.873785 14750 raft_consensus.cc:515] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.874329 14750 leader_election.cc:290] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 11 election: Requested vote from peers 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601), 8152e70912024c1f84634d5e966b4c84 (127.12.15.131:42675)
21:00:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.875041 14688 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d1bb5190405140dfa1141ac6f4c37eb3" candidate_uuid: "5d6f60d583a5412c84b6412358b50165" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "8152e70912024c1f84634d5e966b4c84"
21:00:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.875254 14688 raft_consensus.cc:3060] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 9 FOLLOWER]: Advancing to term 11
21:00:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:57.876256 14433 leader_election.cc:336] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111)
21:00:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.879243 14688 raft_consensus.cc:2468] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d6f60d583a5412c84b6412358b50165 in term 11.
21:00:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.879717 14435 leader_election.cc:304] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5d6f60d583a5412c84b6412358b50165, 8152e70912024c1f84634d5e966b4c84; no voters: 83ca46c2c5f54616bd88be0a9bb0a7b6
21:00:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.880085 14750 raft_consensus.cc:2804] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 11 FOLLOWER]: Leader election won for term 11
21:00:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.880483 14750 raft_consensus.cc:697] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [term 11 LEADER]: Becoming Leader. State: Replica: 5d6f60d583a5412c84b6412358b50165, State: Running, Role: LEADER
21:00:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.881210 14750 consensus_queue.cc:237] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:57.885149 12534 catalog_manager.cc:5696] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 reported cstate change: term changed from 10 to 11, leader changed from 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129) to 5d6f60d583a5412c84b6412358b50165 (127.12.15.130). New cstate: current_term: 11 leader_uuid: "5d6f60d583a5412c84b6412358b50165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } health_report { overall_health: UNKNOWN } } }
21:00:58.089 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet d1bb5190405140dfa1141ac6f4c37eb3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987' with partition [<start>, <end>)
21:00:58.089 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d1bb5190405140dfa1141ac6f4c37eb3@[5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)[L],8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675),83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)]}]
21:00:58.089 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d1bb5190405140dfa1141ac6f4c37eb3, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1469), Traces: [0ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [1ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [2ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [10ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [12ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [12ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [32ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [33ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [34ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [34ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [49ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [50ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [50ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [109ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [110ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [110ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [488ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, [489ms] sending RPC to server 83ca46c2c5f54616bd88be0a9bb0a7b6, [491ms] received response from server 83ca46c2c5f54616bd88be0a9bb0a7b6: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [491ms] delaying RPC due to: Network error: Failed to connect to peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601): Connection refused: /127.12.15.129:46601, [1465ms] refreshing cache from master, [1465ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.15.188:37439, [1468ms] Sub RPC GetTableLocations: received response from server master-127.12.15.188:37439: OK, deferred=Deferred@400158808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fa38036b560@2b57829 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878) -> (continuation of Deferred@1756947225 after retry RPC@2023520490) -> (continuation of Deferred@1819189105 after retry RPC@1999561022) -> (continuation of Deferred@1000681873 after retry RPC@514603788), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29b25a78 -> wakeup thread Time-limited test -> (continuation of Deferred@889148819 after retry RPC@262514926) -> (continuation of Deferred@48642108 after retry RPC@539361878) -> (continuation of Deferred@1756947225 after retry RPC@2023520490) -> (continuation of Deferred@1819189105 after retry RPC@1999561022) -> (continuation of Deferred@1000681873 after retry RPC@514603788))) after lookup
21:00:58.090 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] connecting to peer
21:00:58.091 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] Successfully connected to peer
21:00:58.115 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x31c8d930, L:/127.0.0.1:41708 - R:/127.12.15.130:39631] using TOKEN/null
21:00:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.121515 14688 raft_consensus.cc:1275] T d1bb5190405140dfa1141ac6f4c37eb3 P 8152e70912024c1f84634d5e966b4c84 [term 11 FOLLOWER]: Refusing update from remote peer 5d6f60d583a5412c84b6412358b50165: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
21:00:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.121516 14433 consensus_peers.cc:597] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 -> Peer 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601): Couldn't send request to peer 83ca46c2c5f54616bd88be0a9bb0a7b6. Status: Network error: Client connection negotiation failed: client connection to 127.12.15.129:46601: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.122442 14750 consensus_queue.cc:1048] T d1bb5190405140dfa1141ac6f4c37eb3 P 5d6f60d583a5412c84b6412358b50165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
21:00:58.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.136873 14770 mvcc.cc:204] Tried to move back new op lower bound from 7305398100453756928 to 7305398099484979200. Current Snapshot: MvccSnapshot[applied={T|T < 7305398100453756928}]
21:00:58.157 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.15.129:46601
21:00:58.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.158283 12350 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:00:58.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.508440 14777 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:00:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.508780 14777 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.508865 14777 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:00:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.517268 14777 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:00:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.517439 14777 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.15.129
21:00:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.527616 14777 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.15.148:36847
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.15.129:46601
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.15.129
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42207
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.15.190:41363,127.12.15.189:37857,127.12.15.188:37439
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.15.129
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/logs
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:00:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on bdcb31816ec0
21:00:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16877
21:00:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.529448 14777 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.531397 14777 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.542282 14783 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.542253 14784 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:58.543725 14786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.544267 14777 server_base.cc:1061] running on GCE node
21:00:58.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.545059 14777 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.546231 14777 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.547493 14777 hybrid_clock.cc:648] HybridClock initialized: now 1783544458547424 us; error 65 us; skew 500 ppm
21:00:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.547998 14777 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.550649 14777 webserver.cc:534] Webserver started at http://127.12.15.129:42207/ using document root <none> and password file <none>
21:00:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.551556 14777 fs_manager.cc:362] Metadata directory not provided
21:00:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.551705 14777 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.557837 14777 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
21:00:58.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.561199 14792 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:00:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.562860 14777 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
21:00:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.563071 14777 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data,/tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6"
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:00:18 on dist-test-slave-9w0b"
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.563898 14777 fs_report.cc:389] FS layout report
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/wal
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/data
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:58.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.580171 14777 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:00:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.581184 14777 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.581530 14777 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:58.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.582602 14777 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:00:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.584933 14799 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:00:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.591581 14777 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:00:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.591743 14777 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
21:00:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.591889 14777 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:00:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.595337 14777 ts_tablet_manager.cc:616] Registered 1 tablets
21:00:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.595491 14777 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
21:00:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.596035 14799 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap starting.
21:00:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.625213 14799 log.cc:826] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Log is configured to *not* fsync() on all Append() calls
21:00:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.650034 14777 rpc_server.cc:307] RPC server started. Bound to: 127.12.15.129:46601
21:00:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.650159 14906 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.15.129:46601 every 8 connection(s)
21:00:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.651684 14777 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasksyq7ka/test-tmp/mini-kudu-cluster6016701562945993538/ts-0/data/info.pb
21:00:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.656759 12350 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu as pid 14777
21:00:58.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:00:58.681 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:58.689 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544419987, tablet=d1bb5190405140dfa1141ac6f4c37eb3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:00:58.695 [DEBUG - main] (Connection.java:575) [peer master-127.12.15.189:37857(127.12.15.189:37857)] disconnecting while in state READY
21:00:58.700 [DEBUG - main] (Connection.java:575) [peer 83ca46c2c5f54616bd88be0a9bb0a7b6(127.12.15.129:46601)] disconnecting while in state TERMINATED
21:00:58.700 [DEBUG - main] (Connection.java:575) [peer master-127.12.15.188:37439(127.12.15.188:37439)] disconnecting while in state READY
21:00:58.700 [DEBUG - main] (Connection.java:575) [peer 8152e70912024c1f84634d5e966b4c84(127.12.15.131:42675)] disconnecting while in state TERMINATED
21:00:58.700 [DEBUG - main] (Connection.java:575) [peer master-127.12.15.190:41363(127.12.15.190:41363)] disconnecting while in state READY
21:00:58.700 [DEBUG - main] (Connection.java:575) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] disconnecting while in state READY
21:00:58.702 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.15.189:37857(127.12.15.189:37857)] handling channelInactive
21:00:58.702 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.12.15.190:41363(127.12.15.190:41363)] handling channelInactive
21:00:58.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.15.189:37857(127.12.15.189:37857)] cleaning up while in state READY due to: connection closed
21:00:58.702 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.12.15.190:41363(127.12.15.190:41363)] cleaning up while in state READY due to: connection closed
21:00:58.703 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.15.188:37439(127.12.15.188:37439)] handling channelInactive
21:00:58.703 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.15.188:37439(127.12.15.188:37439)] cleaning up while in state READY due to: connection closed
21:00:58.703 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:00:58.709 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] handling channelInactive
21:00:58.709 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5d6f60d583a5412c84b6412358b50165(127.12.15.130:39631)] cleaning up while in state READY due to: connection closed
21:00:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.709988 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 14777
21:00:58.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.739097 14908 heartbeater.cc:344] Connected to a master server at 127.12.15.190:41363
21:00:58.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.739648 14908 heartbeater.cc:461] Registering TS with master...
21:00:58.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.741014 14908 heartbeater.cc:507] Master 127.12.15.190:41363 requested a full tablet report, sending...
21:00:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.748124 14799 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:00:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.748853 14799 tablet_bootstrap.cc:492] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Bootstrap complete.
21:00:58.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.750449 14799 ts_tablet_manager.cc:1403] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent bootstrapping tablet: real 0.155s	user 0.069s	sys 0.010s
21:00:58.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.755738 12392 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.758147 14907 heartbeater.cc:344] Connected to a master server at 127.12.15.188:37439
21:00:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.758343 14907 heartbeater.cc:461] Registering TS with master...
21:00:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.759053 14907 heartbeater.cc:507] Master 127.12.15.188:37439 requested a full tablet report, sending...
21:00:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.761350 12534 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.762485 14799 raft_consensus.cc:359] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.763586 14799 raft_consensus.cc:740] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83ca46c2c5f54616bd88be0a9bb0a7b6, State: Initialized, Role: FOLLOWER
21:00:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.763806 12534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.15.129:45443
21:00:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.764420 14799 consensus_queue.cc:260] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5d6f60d583a5412c84b6412358b50165" member_type: VOTER last_known_addr { host: "127.12.15.130" port: 39631 } } peers { permanent_uuid: "83ca46c2c5f54616bd88be0a9bb0a7b6" member_type: VOTER last_known_addr { host: "127.12.15.129" port: 46601 } } peers { permanent_uuid: "8152e70912024c1f84634d5e966b4c84" member_type: VOTER last_known_addr { host: "127.12.15.131" port: 42675 } }
21:00:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.767742 14799 ts_tablet_manager.cc:1434] T d1bb5190405140dfa1141ac6f4c37eb3 P 83ca46c2c5f54616bd88be0a9bb0a7b6: Time spent starting tablet: real 0.017s	user 0.012s	sys 0.000s
21:00:58.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.774284 14907 heartbeater.cc:499] Master 127.12.15.188:37439 was elected leader, sending a full tablet report...
21:00:58.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.790748 14912 heartbeater.cc:344] Connected to a master server at 127.12.15.189:37857
21:00:58.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.790956 14912 heartbeater.cc:461] Registering TS with master...
21:00:58.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.791325 14912 heartbeater.cc:507] Master 127.12.15.189:37857 requested a full tablet report, sending...
21:00:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.792856 12462 ts_manager.cc:194] Re-registered known tserver with Master: 83ca46c2c5f54616bd88be0a9bb0a7b6 (127.12.15.129:46601)
21:00:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.793402 14902 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.895802 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 14777
21:00:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.919813 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 14418
21:00:58.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:58.928267 14542 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.046521 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 14418
21:00:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.066320 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 14603
21:00:59.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.074102 14729 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.166821 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 14603
21:00:59.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:00:59.184217 12379 connection.cc:573] server connection from 127.12.15.131:33001 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:59.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.184505 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 12361
21:00:59.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.185918 12422 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.310393 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 12361
21:00:59.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.342550 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 12429
21:00:59.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.343840 12493 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.458590 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 12429
21:00:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.481079 12350 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu pid 12500
21:00:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.482297 12564 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:59.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:00:59.603641 12350 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasksyq7ka/build/asan/bin/kudu with pid 12500
21:00:59.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:00:59Z chronyd exiting

Time: 44.991

OK (1 test)

