JUnit version 4.13.2
.14:22:22.279 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1779114142270
14:22:22.337 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
14:22:22.371 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:22:22.383 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyeLMhy/test-tmp
14:22:22.410 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
14:22:22.413 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyeLMhy/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
14:22:22.636 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

14:22:22.863 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
14:22:22.865 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

14:22:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:22.867880  8257 mini_oidc.cc:150] Starting JWKS server
14:22:22.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:22.870865  8257 webserver.cc:533] Webserver started at http://127.0.0.1:46881/ using document root <none> and password file <none>
14:22:22.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:22.871156  8257 mini_oidc.cc:186] Starting OIDC Discovery server
14:22:22.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:22.871733  8257 webserver.cc:533] Webserver started at http://127.0.0.1:44165/ using document root <none> and password file <none>
14:22:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T14:22:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:22:22.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T14:22:22Z Disabled control of system clock
14:22:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:22.904852  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal
14:22:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data
14:22:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/logs
14:22:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data/info.pb
14:22:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:22:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.126:46859
14:22:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.126
14:22:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:22.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:22.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:22.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46881/jwks.json with env {}
14:22:23.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.267844  8269 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:23.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.268242  8269 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:23.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.268326  8269 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:23.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.277415  8269 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:22:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.277534  8269 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.277590  8269 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:22:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.277631  8269 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:22:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.277688  8269 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:22:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.277779  8269 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46881/jwks.json
14:22:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.291096  8269 master_runner.cc:387] Master server non-default flags:
14:22:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data
14:22:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal
14:22:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.126:46859
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46881/jwks.json
14:22:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data/info.pb
14:22:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.126
14:22:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/logs
14:22:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:22:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:23.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.293610  8269 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:23.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.295939  8269 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.309888  8274 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.310209  8269 server_base.cc:1061] running on GCE node
14:22:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.309883  8277 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.309928  8275 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:23.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.313458  8269 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.315570  8269 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.319136  8269 hybrid_clock.cc:648] HybridClock initialized: now 1779114143319018 us; error 120 us; skew 500 ppm
14:22:23.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.319778  8269 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.323724  8269 webserver.cc:533] Webserver started at http://127.8.16.126:38771/ using document root <none> and password file <none>
14:22:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.324965  8269 fs_manager.cc:362] Metadata directory not provided
14:22:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.325385  8269 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.325953  8269 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:22:23.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.329110  8269 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data/instance:
14:22:23.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7"
14:22:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:23 on dist-test-slave-2txq"
14:22:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.330708  8269 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal/instance:
14:22:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7"
14:22:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:23 on dist-test-slave-2txq"
14:22:23.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.338429  8269 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.001s
14:22:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.343924  8283 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.346225  8269 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.000s
14:22:23.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.346549  8269 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal
14:22:23.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7"
14:22:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:23 on dist-test-slave-2txq"
14:22:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.346879  8269 fs_report.cc:389] FS layout report
14:22:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal
14:22:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal
14:22:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data/data
14:22:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.379550  8269 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.380779  8269 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.381219  8269 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.404626  8269 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.126:46859
14:22:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.404665  8334 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.126:46859 every 8 connection(s)
14:22:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.406572  8269 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/data/info.pb
14:22:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.407697  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 8269
14:22:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.408051  8257 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-0/wal/instance
14:22:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.410903  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal
14:22:23.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data
14:22:23.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:23.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/logs
14:22:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data/info.pb
14:22:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:22:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.125:36273
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.125
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46881/jwks.json with env {}
14:22:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.411845  8335 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.420991  8335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } has no permanent_uuid. Determining permanent_uuid...
14:22:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.442065  8335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } has no permanent_uuid. Determining permanent_uuid...
14:22:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.443694  8284 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.125:36273: connect: Connection refused (error 111)
14:22:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.445634  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.125:36273: Network error: Client connection negotiation failed: client connection to 127.8.16.125:36273: connect: Connection refused (error 111)
14:22:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.495087  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } attempt: 1
14:22:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.497444  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.125:36273: Network error: Client connection negotiation failed: client connection to 127.8.16.125:36273: connect: Connection refused (error 111)
14:22:23.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.565805  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } attempt: 2
14:22:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.568425  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.125:36273: Network error: Client connection negotiation failed: client connection to 127.8.16.125:36273: connect: Connection refused (error 111)
14:22:23.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.659888  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } attempt: 3
14:22:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.662283  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.125:36273: Network error: Client connection negotiation failed: client connection to 127.8.16.125:36273: connect: Connection refused (error 111)
14:22:23.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.790618  8337 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:23.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.791038  8337 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:23.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.791152  8337 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.800547  8337 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:22:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.800730  8337 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.800815  8337 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:22:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.800876  8337 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:22:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.800949  8337 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:22:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.800998  8337 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46881/jwks.json
14:22:23.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.805676  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } attempt: 4
14:22:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.808270  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.125:36273: Network error: Client connection negotiation failed: client connection to 127.8.16.125:36273: connect: Connection refused (error 111)
14:22:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.813186  8337 master_runner.cc:387] Master server non-default flags:
14:22:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data
14:22:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal
14:22:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:23.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.125:36273
14:22:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46881/jwks.json
14:22:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data/info.pb
14:22:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.125
14:22:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/logs
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:22:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.815435  8337 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.818043  8337 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.829448  8345 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.829977  8346 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.832155  8348 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.832491  8337 server_base.cc:1061] running on GCE node
14:22:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.833837  8337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:23.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.836393  8337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:23.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.837692  8337 hybrid_clock.cc:648] HybridClock initialized: now 1779114143837606 us; error 73 us; skew 500 ppm
14:22:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.838187  8337 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.841611  8337 webserver.cc:533] Webserver started at http://127.8.16.125:44937/ using document root <none> and password file <none>
14:22:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.842680  8337 fs_manager.cc:362] Metadata directory not provided
14:22:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.842828  8337 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.843308  8337 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:22:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.847090  8337 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data/instance:
14:22:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "928e8d0a443b49f3ab82030c4d41a92e"
14:22:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:23 on dist-test-slave-2txq"
14:22:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.848523  8337 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal/instance:
14:22:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "928e8d0a443b49f3ab82030c4d41a92e"
14:22:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:23 on dist-test-slave-2txq"
14:22:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.856876  8337 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.009s	sys 0.001s
14:22:23.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.860961  8354 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.862797  8337 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
14:22:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.863018  8337 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal
14:22:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "928e8d0a443b49f3ab82030c4d41a92e"
14:22:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:23 on dist-test-slave-2txq"
14:22:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.863265  8337 fs_report.cc:389] FS layout report
14:22:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal
14:22:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal
14:22:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data/data
14:22:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:23.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.886861  8337 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:23.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.888041  8337 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:23.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.888552  8337 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.914695  8337 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.125:36273
14:22:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.914712  8405 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.125:36273 every 8 connection(s)
14:22:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.916649  8337 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/data/info.pb
14:22:23.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.921393  8406 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.926365  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 8337
14:22:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.926596  8257 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-1/wal/instance
14:22:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.927901  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal
14:22:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data
14:22:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/logs
14:22:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data/info.pb
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.124:41323
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.124
14:22:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46881/jwks.json with env {}
14:22:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.927805  8406 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } has no permanent_uuid. Determining permanent_uuid...
14:22:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.947293  8406 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } has no permanent_uuid. Determining permanent_uuid...
14:22:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:23.962672  8406 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } has no permanent_uuid. Determining permanent_uuid...
14:22:23.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.964303  8358 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:23.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:23.966552  8406 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.016192  8406 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 1
14:22:24.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.019449  8406 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.087749  8406 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 2
14:22:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.090410  8406 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.107610  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } attempt: 5
14:22:24.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.116529  8335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } has no permanent_uuid. Determining permanent_uuid...
14:22:24.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.119043  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.170553  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 1
14:22:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.172837  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.181749  8406 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 3
14:22:24.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.184424  8406 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.241158  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 2
14:22:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.243654  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.327759  8406 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 4
14:22:24.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.330225  8406 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.338776  8408 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:24.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.339128  8408 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:24.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.339207  8408 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:24.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.349916  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 3
14:22:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.352763  8335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.16.124:41323: Network error: Client connection negotiation failed: client connection to 127.8.16.124:41323: connect: Connection refused (error 111)
14:22:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.353582  8408 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:22:24.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.353672  8408 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:24.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.353730  8408 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:22:24.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.353827  8408 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:22:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.353888  8408 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:22:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.353929  8408 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46881/jwks.json
14:22:24.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.366238  8408 master_runner.cc:387] Master server non-default flags:
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.124:41323
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46881/jwks.json
14:22:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data/info.pb
14:22:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.124
14:22:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/logs
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:22:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.368110  8408 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.370359  8408 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.381065  8416 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.381044  8417 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.381518  8419 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.382529  8408 server_base.cc:1061] running on GCE node
14:22:24.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.383499  8408 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.385195  8408 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.386543  8408 hybrid_clock.cc:648] HybridClock initialized: now 1779114144386466 us; error 70 us; skew 500 ppm
14:22:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.386915  8408 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.389863  8408 webserver.cc:533] Webserver started at http://127.8.16.124:33451/ using document root <none> and password file <none>
14:22:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.390750  8408 fs_manager.cc:362] Metadata directory not provided
14:22:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.390857  8408 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.391180  8408 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:22:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.394008  8408 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data/instance:
14:22:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2bff06a42474154be9232a7020cee9e"
14:22:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:24 on dist-test-slave-2txq"
14:22:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.394824  8408 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal/instance:
14:22:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2bff06a42474154be9232a7020cee9e"
14:22:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:24 on dist-test-slave-2txq"
14:22:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.401139  8408 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
14:22:24.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.405778  8425 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.407684  8408 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
14:22:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.407945  8408 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal
14:22:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2bff06a42474154be9232a7020cee9e"
14:22:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:24 on dist-test-slave-2txq"
14:22:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.408219  8408 fs_report.cc:389] FS layout report
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data/data
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.442015  8408 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.443243  8408 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.443634  8408 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.467440  8408 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.124:41323
14:22:24.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.467473  8476 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.124:41323 every 8 connection(s)
14:22:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.469347  8408 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/data/info.pb
14:22:24.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.474795  8477 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.477563  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 8408
14:22:24.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.477815  8257 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/master-2/wal/instance
14:22:24.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.483831  8477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } has no permanent_uuid. Determining permanent_uuid...
14:22:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.503240  8477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } has no permanent_uuid. Determining permanent_uuid...
14:22:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.511112  8477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } has no permanent_uuid. Determining permanent_uuid...
14:22:24.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.524606  8477 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: Bootstrap starting.
14:22:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.529457  8477 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: Neither blocks nor log segments found. Creating new log.
14:22:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.530223  8335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 4
14:22:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.531908  8477 log.cc:826] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: Log is configured to *not* fsync() on all Append() calls
14:22:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.535392  8477 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: No bootstrap required, opened a new log
14:22:24.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.542253  8477 raft_consensus.cc:359] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.542654  8477 raft_consensus.cc:385] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:22:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.542774  8477 raft_consensus.cc:740] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e2bff06a42474154be9232a7020cee9e, State: Initialized, Role: FOLLOWER
14:22:24.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.544207  8335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: Bootstrap starting.
14:22:24.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.544477  8477 consensus_queue.cc:260] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.546239  8487 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.546586  8487 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.548305  8477 sys_catalog.cc:565] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: configured and running, proceeding with master startup.
14:22:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.549116  8335 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: Neither blocks nor log segments found. Creating new log.
14:22:24.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.550668  8335 log.cc:826] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: Log is configured to *not* fsync() on all Append() calls
14:22:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.555199  8335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: No bootstrap required, opened a new log
14:22:24.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.562875  8335 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.565874  8335 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:22:24.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.566349  8335 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3af4d0e6ddaa4d31913cbaef4c5407d7, State: Initialized, Role: FOLLOWER
14:22:24.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.566382  8499 catalog_manager.cc:1594] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:22:24.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.566531  8499 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:22:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.558979  8408 master_runner.cc:187] Error getting master registration for 127.8.16.126:46859: OK, instance_id { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" instance_seqno: 1779114143398524 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:22:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.567724  8335 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.568225  8408 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:22:24.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.569087  8501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.569504  8501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.570765  8335 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: configured and running, proceeding with master startup.
14:22:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.584817  8269 master_runner.cc:187] Error getting master registration for 127.8.16.125:36273: OK, instance_id { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" instance_seqno: 1779114143907780 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:22:24.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.590613  8269 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:22:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.596307  8512 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:22:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.597054  8512 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:22:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.629537  8406 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } attempt: 5
14:22:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.635213  8289 tablet.cc:2404] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:22:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.645367  8406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e: Bootstrap starting.
14:22:24.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.650092  8406 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e: Neither blocks nor log segments found. Creating new log.
14:22:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.652626  8406 log.cc:826] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e: Log is configured to *not* fsync() on all Append() calls
14:22:24.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.658627  8406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e: No bootstrap required, opened a new log
14:22:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.668028  8406 raft_consensus.cc:359] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.669106  8406 raft_consensus.cc:385] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:22:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.669612  8406 raft_consensus.cc:740] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 928e8d0a443b49f3ab82030c4d41a92e, State: Initialized, Role: FOLLOWER
14:22:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.671059  8406 consensus_queue.cc:260] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.672704  8515 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.673480  8515 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.674623  8406 sys_catalog.cc:565] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: configured and running, proceeding with master startup.
14:22:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.696650  8431 tablet.cc:2404] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:22:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.700766  8515 raft_consensus.cc:493] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.701512  8515 raft_consensus.cc:515] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.703634  8515 leader_election.cc:290] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3af4d0e6ddaa4d31913cbaef4c5407d7 (127.8.16.126:46859), e2bff06a42474154be9232a7020cee9e (127.8.16.124:41323)
14:22:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.704952  8337 master_runner.cc:264] No leader master found from master 928e8d0a443b49f3ab82030c4d41a92e
14:22:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.705081  8337 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:22:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.707590  8310 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928e8d0a443b49f3ab82030c4d41a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" is_pre_election: true
14:22:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.708196  8310 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 928e8d0a443b49f3ab82030c4d41a92e in term 0.
14:22:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.711767  8452 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928e8d0a443b49f3ab82030c4d41a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e2bff06a42474154be9232a7020cee9e" is_pre_election: true
14:22:24.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.712751  8452 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 928e8d0a443b49f3ab82030c4d41a92e in term 0.
14:22:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.720836  8358 leader_election.cc:304] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 928e8d0a443b49f3ab82030c4d41a92e, e2bff06a42474154be9232a7020cee9e; no voters: 
14:22:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.722405  8515 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Leader pre-election won for term 1
14:22:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.722635  8515 raft_consensus.cc:493] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:22:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.722740  8515 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 0 FOLLOWER]: Advancing to term 1
14:22:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.726626  8515 raft_consensus.cc:515] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.727581  8515 leader_election.cc:290] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [CANDIDATE]: Term 1 election: Requested vote from peers 3af4d0e6ddaa4d31913cbaef4c5407d7 (127.8.16.126:46859), e2bff06a42474154be9232a7020cee9e (127.8.16.124:41323)
14:22:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.730593  8310 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928e8d0a443b49f3ab82030c4d41a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7"
14:22:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.730888  8310 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 0 FOLLOWER]: Advancing to term 1
14:22:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.742976  8452 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928e8d0a443b49f3ab82030c4d41a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e2bff06a42474154be9232a7020cee9e"
14:22:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.743270  8452 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 0 FOLLOWER]: Advancing to term 1
14:22:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.747299  8452 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 928e8d0a443b49f3ab82030c4d41a92e in term 1.
14:22:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.747958  8358 leader_election.cc:304] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 928e8d0a443b49f3ab82030c4d41a92e, e2bff06a42474154be9232a7020cee9e; no voters: 
14:22:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.750734  8515 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 1 FOLLOWER]: Leader election won for term 1
14:22:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.751811  8310 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 928e8d0a443b49f3ab82030c4d41a92e in term 1.
14:22:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.764354  8526 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:22:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:24.765434  8526 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.765539  8515 raft_consensus.cc:697] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [term 1 LEADER]: Becoming Leader. State: Replica: 928e8d0a443b49f3ab82030c4d41a92e, State: Running, Role: LEADER
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.766731  8515 consensus_queue.cc:237] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } }
14:22:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.791764  8528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 928e8d0a443b49f3ab82030c4d41a92e. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.792151  8528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: This master's current role is: LEADER
14:22:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.803453  8530 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:22:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.806785  8530 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:22:24.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.827899  8452 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [term 1 FOLLOWER]: Refusing update from remote peer 928e8d0a443b49f3ab82030c4d41a92e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:22:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.830222  8528 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:22:24.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.835969  8310 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [term 1 FOLLOWER]: Refusing update from remote peer 928e8d0a443b49f3ab82030c4d41a92e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:22:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.837713  8528 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [LEADER]: Connected to new peer: Peer: permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:22:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.866271  8501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 928e8d0a443b49f3ab82030c4d41a92e. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.866664  8501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.869494  8531 mvcc.cc:204] Tried to move back new op lower bound from 7287251537201917952 to 7287251536970637312. Current Snapshot: MvccSnapshot[applied={T|T < 7287251537201917952}]
14:22:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.872001  8487 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 928e8d0a443b49f3ab82030c4d41a92e. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.872368  8487 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.881863  8487 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.882442  8487 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.884160  8532 mvcc.cc:204] Tried to move back new op lower bound from 7287251537201917952 to 7287251536970637312. Current Snapshot: MvccSnapshot[applied={T|T < 7287251537201917952}]
14:22:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.884275  8533 mvcc.cc:204] Tried to move back new op lower bound from 7287251537201917952 to 7287251536970637312. Current Snapshot: MvccSnapshot[applied={T|T < 7287251537201917952}]
14:22:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.887754  8501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.887993  8501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7 [sys.catalog]: This master's current role is: FOLLOWER
14:22:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.889335  8528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.889847  8528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: This master's current role is: LEADER
14:22:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.890128  8528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "928e8d0a443b49f3ab82030c4d41a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af4d0e6ddaa4d31913cbaef4c5407d7" member_type: VOTER last_known_addr { host: "127.8.16.126" port: 46859 } } peers { permanent_uuid: "928e8d0a443b49f3ab82030c4d41a92e" member_type: VOTER last_known_addr { host: "127.8.16.125" port: 36273 } } peers { permanent_uuid: "e2bff06a42474154be9232a7020cee9e" member_type: VOTER last_known_addr { host: "127.8.16.124" port: 41323 } } }
14:22:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.890305  8528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e [sys.catalog]: This master's current role is: LEADER
14:22:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.892943  8530 catalog_manager.cc:1383] Generated new cluster ID: 6d762ddf7cbd436f84a4bf98d3f2205e
14:22:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.893072  8530 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:22:24.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.978540  8530 catalog_manager.cc:1406] Generated new certificate authority record
14:22:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:24.980813  8530 catalog_manager.cc:1540] Loading token signing keys...
14:22:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.016435  8530 catalog_manager.cc:6070] T 00000000000000000000000000000000 P 928e8d0a443b49f3ab82030c4d41a92e: Generated new TSK 0
14:22:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.018921  8530 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:22:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.061980  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:0
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:25.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:25.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.568915  8499 catalog_manager.cc:1295] Loaded cluster ID: 6d762ddf7cbd436f84a4bf98d3f2205e
14:22:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.569181  8499 catalog_manager.cc:1588] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: loading cluster ID for follower catalog manager: success
14:22:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.570294  8540 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.570600  8540 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.570675  8540 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.572808  8499 catalog_manager.cc:1610] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: acquiring CA information for follower catalog manager: success
14:22:25.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.575105  8499 catalog_manager.cc:1638] T 00000000000000000000000000000000 P e2bff06a42474154be9232a7020cee9e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:22:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.584239  8540 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.584537  8540 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.65
14:22:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.599050  8540 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:0
14:22:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.599844  8512 catalog_manager.cc:1295] Loaded cluster ID: 6d762ddf7cbd436f84a4bf98d3f2205e
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.599938  8512 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: loading cluster ID for follower catalog manager: success
14:22:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.601583  8540 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.604030  8512 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: acquiring CA information for follower catalog manager: success
14:22:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.604084  8540 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.606791  8512 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 3af4d0e6ddaa4d31913cbaef4c5407d7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:22:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.618027  8545 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.618028  8546 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:25.618027  8548 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.619998  8540 server_base.cc:1061] running on GCE node
14:22:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.620764  8540 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.622171  8540 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.625655  8540 hybrid_clock.cc:648] HybridClock initialized: now 1779114145625570 us; error 59 us; skew 500 ppm
14:22:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.626309  8540 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.631304  8540 webserver.cc:533] Webserver started at http://127.8.16.65:34929/ using document root <none> and password file <none>
14:22:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.632541  8540 fs_manager.cc:362] Metadata directory not provided
14:22:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.632718  8540 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.633291  8540 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:22:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.636197  8540 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/instance:
14:22:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:25 on dist-test-slave-2txq"
14:22:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.637002  8540 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal/instance:
14:22:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:25 on dist-test-slave-2txq"
14:22:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.643602  8540 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.004s	sys 0.000s
14:22:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.648305  8554 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.650848  8540 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.000s
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.651103  8540 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:25 on dist-test-slave-2txq"
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.651382  8540 fs_report.cc:389] FS layout report
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/data
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:25.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.670218  8540 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.671381  8540 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.671850  8540 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.673187  8540 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.675526  8540 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:22:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.675668  8540 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:22:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.675781  8540 ts_tablet_manager.cc:616] Registered 0 tablets
14:22:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.675853  8540 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:22:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.726742  8666 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.65:35141 every 8 connection(s)
14:22:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.726941  8540 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.65:35141
14:22:25.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.728652  8540 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.730650  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 8540
14:22:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.731252  8257 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal/instance
14:22:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.740985  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:0
14:22:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.764950  8667 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.764971  8669 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.764972  8668 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.765938  8668 heartbeater.cc:461] Registering TS with master...
14:22:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.765960  8667 heartbeater.cc:461] Registering TS with master...
14:22:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.767516  8667 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.767558  8668 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.767572  8669 heartbeater.cc:461] Registering TS with master...
14:22:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.768283  8669 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.770586  8300 ts_manager.cc:194] Registered new tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.770596  8441 ts_manager.cc:194] Registered new tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.770612  8371 ts_manager.cc:194] Registered new tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:25.772795  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.65:41663
14:22:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.159633  8675 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.160049  8675 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.160156  8675 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:26.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.170248  8675 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:26.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.170518  8675 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.66
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.181952  8675 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:0
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.184270  8675 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.187077  8675 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.199954  8682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.201026  8675 server_base.cc:1061] running on GCE node
14:22:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.199801  8683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.199801  8685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.202603  8675 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:26.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.203789  8675 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.207378  8675 hybrid_clock.cc:648] HybridClock initialized: now 1779114146207294 us; error 106 us; skew 500 ppm
14:22:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.207787  8675 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.211282  8675 webserver.cc:533] Webserver started at http://127.8.16.66:35077/ using document root <none> and password file <none>
14:22:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.212090  8675 fs_manager.cc:362] Metadata directory not provided
14:22:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.212211  8675 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.212549  8675 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:22:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.217350  8675 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/instance:
14:22:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.219269  8675 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal/instance:
14:22:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:26.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.226394  8675 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.001s
14:22:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.231526  8691 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.233330  8675 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
14:22:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.233510  8675 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.233904  8675 fs_report.cc:389] FS layout report
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/data
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.268061  8675 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.268955  8675 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.269220  8675 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:26.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.270392  8675 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:26.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.272233  8675 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:22:26.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.272332  8675 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:22:26.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.272432  8675 ts_tablet_manager.cc:616] Registered 0 tablets
14:22:26.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.272526  8675 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:22:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.326264  8675 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.66:42745
14:22:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.326412  8804 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.66:42745 every 8 connection(s)
14:22:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.327982  8675 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.330626  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 8675
14:22:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.330910  8257 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal/instance
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.338119  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:0
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:26.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:26.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:26.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.363449  8805 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.363504  8807 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.363966  8805 heartbeater.cc:461] Registering TS with master...
14:22:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.364058  8807 heartbeater.cc:461] Registering TS with master...
14:22:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.365128  8807 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.365154  8805 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.367406  8806 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.367981  8806 heartbeater.cc:461] Registering TS with master...
14:22:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.368798  8806 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.369555  8371 ts_manager.cc:194] Registered new tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.370859  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.66:36979
14:22:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.371097  8300 ts_manager.cc:194] Registered new tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.371603  8441 ts_manager.cc:194] Registered new tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:26.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.776340  8669 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.813627  8813 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.814013  8813 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.814116  8813 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.824743  8813 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.825059  8813 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.67
14:22:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.835772  8813 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:0
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.838507  8813 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:26.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.840963  8813 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.853766  8820 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.853766  8821 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:26.853811  8823 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.854771  8813 server_base.cc:1061] running on GCE node
14:22:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.855475  8813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:26.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.858196  8813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:26.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.859488  8813 hybrid_clock.cc:648] HybridClock initialized: now 1779114146859411 us; error 65 us; skew 500 ppm
14:22:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.860049  8813 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.863224  8813 webserver.cc:533] Webserver started at http://127.8.16.67:41293/ using document root <none> and password file <none>
14:22:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.864219  8813 fs_manager.cc:362] Metadata directory not provided
14:22:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.864333  8813 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.864722  8813 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:22:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.867501  8813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/instance:
14:22:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.868395  8813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal/instance:
14:22:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.876340  8813 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.005s
14:22:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.882570  8829 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.884393  8813 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.884651  8813 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.884997  8813 fs_report.cc:389] FS layout report
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/data
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:26.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.910930  8813 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.912045  8813 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.912421  8813 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.914229  8813 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.916189  8813 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:22:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.916324  8813 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:22:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.916435  8813 ts_tablet_manager.cc:616] Registered 0 tablets
14:22:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.916512  8813 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:22:26.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.962426  8813 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.67:42073
14:22:26.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.962551  8941 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.67:42073 every 8 connection(s)
14:22:26.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.964157  8813 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:26.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.966329  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 8813
14:22:26.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.966495  8257 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal/instance
14:22:26.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.992647  8942 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.992746  8943 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.993165  8943 heartbeater.cc:461] Registering TS with master...
14:22:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.993170  8942 heartbeater.cc:461] Registering TS with master...
14:22:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.993763  8944 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.993918  8944 heartbeater.cc:461] Registering TS with master...
14:22:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.995118  8943 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.995260  8944 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.995254  8942 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.997990  8441 ts_manager.cc:194] Registered new tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.998029  8300 ts_manager.cc:194] Registered new tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.998521  8371 ts_manager.cc:194] Registered new tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:26.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:26.999606  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.67:32835
14:22:27.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:27.007234  8257 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:22:27.009 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
14:22:27.010 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

14:22:27.026 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.8.16.126"
      port: 46859
    }
    bound_http_address {
      host: "127.8.16.126"
      port: 38771
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.8.16.125"
      port: 36273
    }
    bound_http_address {
      host: "127.8.16.125"
      port: 44937
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.8.16.124"
      port: 41323
    }
    bound_http_address {
      host: "127.8.16.124"
      port: 33451
    }
  }
}

14:22:27.031 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

14:22:27.034 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.8.16.65"
      port: 35141
    }
    bound_http_address {
      host: "127.8.16.65"
      port: 34929
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.8.16.66"
      port: 42745
    }
    bound_http_address {
      host: "127.8.16.66"
      port: 35077
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.8.16.67"
      port: 42073
    }
    bound_http_address {
      host: "127.8.16.67"
      port: 41293
    }
  }
}

14:22:27.034 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:22:27.244 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:22:27.292 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
14:22:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:27.375109  8807 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:27.504 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.16.125:36273(127.8.16.125:36273)] connecting to peer
14:22:27.573 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.16.126:46859(127.8.16.126:46859)] connecting to peer
14:22:27.588 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.16.124:41323(127.8.16.124:41323)] connecting to peer
14:22:27.597 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.16.126:46859(127.8.16.126:46859)] Successfully connected to peer
14:22:27.601 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.16.125:36273(127.8.16.125:36273)] Successfully connected to peer
14:22:27.603 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.8.16.124:41323(127.8.16.124:41323)] Successfully connected to peer
14:22:27.730 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.16.124
14:22:27.733 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.16.126
14:22:27.733 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.16.125
14:22:27.991 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x967562d7, L:/127.0.0.1:35734 - R:/127.8.16.125:36273] using SASL/PLAIN
14:22:27.991 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x1ec5ec6d, L:/127.0.0.1:47448 - R:/127.8.16.124:41323] using SASL/PLAIN
14:22:27.992 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x74f10c14, L:/127.0.0.1:41984 - R:/127.8.16.126:46859] using SASL/PLAIN
14:22:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.003098  8944 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:28.030 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:22:28.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=576), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.16.125:36273, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.16.126:46859, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.16.124:41323, [545ms] Sub RPC ConnectToMaster: received response from server master-127.8.16.126:46859: OK, [545ms] Sub RPC ConnectToMaster: received response from server master-127.8.16.124:41323: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.8.16.125:36273: OK, deferred=null) after lookup
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.079927  8371 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35734:
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434"
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:22:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
14:22:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:22:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:22:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:28.085700  8371 catalog_manager.cc:7059] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:22:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.161649  8740 tablet_service.cc:1511] Processing CreateTablet for tablet 81030604361040b5ac097199a6d3bcdc (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434 [id=95427fb927634f4381bd74cdad108b78]), partition=RANGE (key) PARTITION UNBOUNDED
14:22:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.164011  8740 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81030604361040b5ac097199a6d3bcdc. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:28.165 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 95427fb927634f4381bd74cdad108b78
14:22:28.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.165352  8602 tablet_service.cc:1511] Processing CreateTablet for tablet 81030604361040b5ac097199a6d3bcdc (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434 [id=95427fb927634f4381bd74cdad108b78]), partition=RANGE (key) PARTITION UNBOUNDED
14:22:28.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.167579  8602 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81030604361040b5ac097199a6d3bcdc. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.184198  8877 tablet_service.cc:1511] Processing CreateTablet for tablet 81030604361040b5ac097199a6d3bcdc (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434 [id=95427fb927634f4381bd74cdad108b78]), partition=RANGE (key) PARTITION UNBOUNDED
14:22:28.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.187050  8978 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap starting.
14:22:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.194036  8977 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap starting.
14:22:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.195863  8877 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81030604361040b5ac097199a6d3bcdc. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.197398  8978 tablet_bootstrap.cc:654] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Neither blocks nor log segments found. Creating new log.
14:22:28.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.198669  8977 tablet_bootstrap.cc:654] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Neither blocks nor log segments found. Creating new log.
14:22:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.199066  8978 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Log is configured to *not* fsync() on all Append() calls
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.200481  8977 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Log is configured to *not* fsync() on all Append() calls
14:22:28.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.207551  8977 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: No bootstrap required, opened a new log
14:22:28.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.208437  8977 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent bootstrapping tablet: real 0.015s	user 0.008s	sys 0.004s
14:22:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.219276  8977 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.221086  8977 raft_consensus.cc:385] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:22:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.221449  8977 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Initialized, Role: FOLLOWER
14:22:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.221444  8978 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: No bootstrap required, opened a new log
14:22:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.221953  8978 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent bootstrapping tablet: real 0.035s	user 0.008s	sys 0.016s
14:22:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.230294  8977 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.231936  8981 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap starting.
14:22:28.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.234831  8977 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent starting tablet: real 0.026s	user 0.012s	sys 0.000s
14:22:28.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.236146  8978 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.238831  8978 raft_consensus.cc:385] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:22:28.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.239014  8978 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Initialized, Role: FOLLOWER
14:22:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.239917  8978 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.245400  8981 tablet_bootstrap.cc:654] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Neither blocks nor log segments found. Creating new log.
14:22:28.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.247763  8981 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Log is configured to *not* fsync() on all Append() calls
14:22:28.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.250167  8978 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent starting tablet: real 0.028s	user 0.015s	sys 0.005s
14:22:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.253376  8981 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: No bootstrap required, opened a new log
14:22:28.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.254079  8981 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent bootstrapping tablet: real 0.022s	user 0.010s	sys 0.000s
14:22:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.260957  8981 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.261507  8981 raft_consensus.cc:385] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:22:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.261615  8981 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Initialized, Role: FOLLOWER
14:22:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.262570  8981 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.264391  8981 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.004s
14:22:28.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.337184  8982 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.337623  8982 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:28.337978  8808 tablet.cc:2404] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:22:28.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.340371  8982 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.350389  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.350667  8897 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.351081  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 0.
14:22:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.351251  8897 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 0.
14:22:28.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.352169  8693 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: 
14:22:28.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.352877  8982 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:22:28.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.353111  8982 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:22:28.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.353209  8982 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 0 FOLLOWER]: Advancing to term 1
14:22:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.355495  8982 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.356178  8982 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 1 election: Requested vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.356803  8897 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.357020  8897 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 0 FOLLOWER]: Advancing to term 1
14:22:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.357041  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.357312  8622 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 0 FOLLOWER]: Advancing to term 1
14:22:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.360139  8897 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 1.
14:22:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.360484  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 1.
14:22:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.360726  8692 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 
14:22:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.361155  8982 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 1 FOLLOWER]: Leader election won for term 1
14:22:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.362267  8982 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 1 LEADER]: Becoming Leader. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Running, Role: LEADER
14:22:28.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.362874  8982 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:28.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.369253  8371 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 reported cstate change: term changed from 0 to 1, leader changed from <none> to d4207c6141b143c590369fc8fef6b316 (127.8.16.66). New cstate: current_term: 1 leader_uuid: "d4207c6141b143c590369fc8fef6b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:28.465 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 95427fb927634f4381bd74cdad108b78
14:22:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:28.471819  8945 tablet.cc:2404] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:22:28.487 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:28.486937  8670 tablet.cc:2404] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:22:28.494 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:28.496 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [10ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=null) after lookup
14:22:28.506 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:28.508 [DEBUG - kudu-nio-3] (Connection.java:757) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] Successfully connected to peer
14:22:28.592 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x90cf2c1d, L:/127.0.0.1:38150 - R:/127.8.16.66:42745] using TOKEN/null
14:22:28.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.607702  8897 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Refusing update from remote peer d4207c6141b143c590369fc8fef6b316: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:22:28.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.607702  8622 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Refusing update from remote peer d4207c6141b143c590369fc8fef6b316: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:22:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.609051  8982 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:22:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.609550  8988 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:22:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.646427  8992 mvcc.cc:204] Tried to move back new op lower bound from 7287251552683495424 to 7287251551698534400. Current Snapshot: MvccSnapshot[applied={T|T < 7287251552683495424}]
14:22:28.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.653869  8993 mvcc.cc:204] Tried to move back new op lower bound from 7287251552683495424 to 7287251551698534400. Current Snapshot: MvccSnapshot[applied={T|T < 7287251552683495424}]
14:22:28.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.664510  8994 mvcc.cc:204] Tried to move back new op lower bound from 7287251552683495424 to 7287251551698534400. Current Snapshot: MvccSnapshot[applied={T|T < 7287251552683495424}]
14:22:28.787 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:28.789 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.66:42745
14:22:28.790 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.791399  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 8675
14:22:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.805845  8800 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:28.913305  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 8675
14:22:28.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:28.946 [DEBUG - kudu-nio-3] (Connection.java:239) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] handling channelInactive
14:22:28.946 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state READY due to: connection closed
14:22:28.946 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: connection closed
14:22:28.961 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:28.961 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:28.962 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test)) after lookup
14:22:28.962 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:28.965 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:28.965 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:28.965 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:28.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:28.981 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:28.981 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [22ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [22ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252))) after lookup
14:22:28.982 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:28.984 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:28.984 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:28.984 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:29.021 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:29.023 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [22ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [22ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [38ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [41ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [77ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705))) after lookup
14:22:29.024 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:29.026 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.026 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.026 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.102 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:29.103 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:29.103 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=159), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [22ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [22ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [38ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [41ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [77ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [80ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [159ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219))) after lookup
14:22:29.104 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:29.106 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.106 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.106 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.520 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:29.520 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:29.521 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=577), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [22ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [22ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [38ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [41ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [77ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [80ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [159ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [161ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [163ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [163ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [573ms] refreshing cache from master, [574ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [576ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219) -> (continuation of Deferred@1470496510 after retry RPC@1406013682), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219) -> (continuation of Deferred@1470496510 after retry RPC@1406013682))) after lookup
14:22:29.521 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:29.522 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.522 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:29.523 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.211915  9009 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.212265  9010 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.212229  9009 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.212420  9010 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.214020  9010 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:30.215536  8555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.216952  9009 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.228474  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:30.228662  8555 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:30.228601  8830 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.228811  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 1.
14:22:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.229454  8831 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 
14:22:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.229581  8897 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.229847  8897 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 1.
14:22:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.231681  9009 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:22:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.231905  8555 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.232384  9009 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.232527  9009 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 1 FOLLOWER]: Advancing to term 2
14:22:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.232862  9010 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Leader pre-election won for term 2
14:22:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.233006  9010 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Starting leader election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:30.233026  8830 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.233107  9010 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 1 FOLLOWER]: Advancing to term 2
14:22:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.237044  9010 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.237416  9009 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.237942  9010 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 2 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.238088  9009 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 2 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.238924  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.239131  8897 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.239274  8622 raft_consensus.cc:2393] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in current term 2: Already voted for candidate 84e8522e549f4f23b3fd7f83497bb3ab in this term.
14:22:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.239379  8897 raft_consensus.cc:2393] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 84e8522e549f4f23b3fd7f83497bb3ab in current term 2: Already voted for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in this term.
14:22:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:30.240757  8555 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:30.240751  8830 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.240880  8830 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316
14:22:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.241093  8555 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab; no voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.241185  9009 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
14:22:30.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:30.241907  9010 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
14:22:31.382 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:31.382 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:31.383 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2439), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [22ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [22ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [38ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [41ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [77ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [80ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [159ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [161ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [163ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [163ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [573ms] refreshing cache from master, [574ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [576ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [578ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [580ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [580ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2434ms] refreshing cache from master, [2434ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [2438ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219) -> (continuation of Deferred@1470496510 after retry RPC@1406013682) -> (continuation of Deferred@1226912678 after retry RPC@1647906826), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219) -> (continuation of Deferred@1470496510 after retry RPC@1406013682) -> (continuation of Deferred@1226912678 after retry RPC@1647906826))) after lookup
14:22:31.384 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:31.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:31.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:31.386 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:31.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.788369  9019 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:31.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.788686  9019 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:31.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.789584  9019 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.790343  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.790598  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 2.
14:22:31.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.791404  8831 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 
14:22:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.791951  9019 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:22:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.792158  9019 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:22:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.792309  9019 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 2 FOLLOWER]: Advancing to term 3
14:22:31.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:31.793143  8830 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:31.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.794734  9019 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:31.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.795393  9019 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 3 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:31.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.796264  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:31.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.796540  8622 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 2 FOLLOWER]: Advancing to term 3
14:22:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:31.797330  8830 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.798862  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 3.
14:22:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.799401  8831 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.799808  9019 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 FOLLOWER]: Leader election won for term 3
14:22:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.800468  9019 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 LEADER]: Becoming Leader. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Running, Role: LEADER
14:22:31.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.801278  9019 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:31.805450  8370 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 reported cstate change: term changed from 1 to 3, leader changed from d4207c6141b143c590369fc8fef6b316 (127.8.16.66) to df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67). New cstate: current_term: 3 leader_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: HEALTHY } } }
14:22:32.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:32.345124  8830 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:32.387305  8622 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Refusing update from remote peer df3bbabb6b104a5e8e63dca4a47eb6b1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
14:22:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:32.388264  9035 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:22:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:34.739677  8830 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:22:35.261 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:35.262 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:35.263 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=6318), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [3ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [22ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [22ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [38ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [41ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [77ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [80ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [159ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [161ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [163ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [163ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [573ms] refreshing cache from master, [574ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [576ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [578ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [580ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [580ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2434ms] refreshing cache from master, [2434ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [2438ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [2440ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2443ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2443ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [6314ms] refreshing cache from master, [6314ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [6318ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1646884322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5abf65ac -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219) -> (continuation of Deferred@1470496510 after retry RPC@1406013682) -> (continuation of Deferred@1226912678 after retry RPC@1647906826) -> (continuation of Deferred@1463446446 after retry RPC@2108266499), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73caf2bb -> wakeup thread Time-limited test -> (continuation of Deferred@930940722 after retry RPC@2082653252) -> (continuation of Deferred@1951542800 after retry RPC@178003705) -> (continuation of Deferred@77348031 after retry RPC@1602060219) -> (continuation of Deferred@1470496510 after retry RPC@1406013682) -> (continuation of Deferred@1226912678 after retry RPC@1647906826) -> (continuation of Deferred@1463446446 after retry RPC@2108266499))) after lookup
14:22:35.264 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:35.265 [DEBUG - kudu-nio-2] (Connection.java:757) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] Successfully connected to peer
14:22:35.290 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x868ec0de, L:/127.0.0.1:34526 - R:/127.8.16.67:42073] using TOKEN/null
14:22:35.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.328637  8830 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111) [suppressed 9 similar messages]
14:22:35.334 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.66:42745
14:22:35.336 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.336983  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.736212  9061 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.736624  9061 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.736743  9061 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.746078  9061 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.746407  9061 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.66
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.757797  9061 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.759968  9061 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.762228  9061 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.775154  9066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.775115  9061 server_base.cc:1061] running on GCE node
14:22:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.775089  9067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:35.775701  9069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.776544  9061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.777771  9061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.779052  9061 hybrid_clock.cc:648] HybridClock initialized: now 1779114155778959 us; error 45 us; skew 500 ppm
14:22:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.779538  9061 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.782162  9061 webserver.cc:533] Webserver started at http://127.8.16.66:35077/ using document root <none> and password file <none>
14:22:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.782945  9061 fs_manager.cc:362] Metadata directory not provided
14:22:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.783171  9061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:35.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.789470  9061 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.007s	sys 0.000s
14:22:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.793269  9075 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.796432  9061 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.004s	sys 0.000s
14:22:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.796665  9061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.797824  9061 fs_report.cc:389] FS layout report
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/data
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.816854  9061 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.818025  9061 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.818377  9061 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:35.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.819725  9061 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:35.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.822458  9082 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.829386  9061 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.829514  9061 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.000s	sys 0.001s
14:22:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.829627  9061 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.833714  9061 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.833846  9061 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.004s
14:22:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.834666  9082 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap starting.
14:22:35.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.858839  9082 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Log is configured to *not* fsync() on all Append() calls
14:22:35.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.879458  9082 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:35.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.880304  9082 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap complete.
14:22:35.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.881778  9082 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent bootstrapping tablet: real 0.048s	user 0.037s	sys 0.009s
14:22:35.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.889596  9082 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:35.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.890718  9082 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Initialized, Role: FOLLOWER
14:22:35.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.891685  9082 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.893772  9082 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent starting tablet: real 0.012s	user 0.012s	sys 0.000s
14:22:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.901192  9061 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.66:42745
14:22:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.901319  9191 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.66:42745 every 8 connection(s)
14:22:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.902684  9061 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.912658  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 9061
14:22:35.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.928192  9193 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.928742  9193 heartbeater.cc:461] Registering TS with master...
14:22:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.930194  9193 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.933612  9192 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.933818  9192 heartbeater.cc:461] Registering TS with master...
14:22:35.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.933962  8442 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:35.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.934494  9192 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.937098  8299 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:35.943 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:35.944 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.67:42073
14:22:35.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:22:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.944870  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 8813
14:22:35.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.948757  9194 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:35.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.948984  9194 heartbeater.cc:461] Registering TS with master...
14:22:35.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.949662  9194 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:35.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.952200  8370 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:35.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.954515  8370 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.66:57253
14:22:35.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.960472  9145 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 1 FOLLOWER]: Advancing to term 3
14:22:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.967128  8937 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:35.967438  9194 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:36.076874  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 8813
14:22:36.101 [DEBUG - kudu-nio-2] (Connection.java:239) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] handling channelInactive
14:22:36.101 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:36.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state READY due to: connection closed
14:22:36.102 [DEBUG - Time-limited test] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:36.103 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.103 [DEBUG - kudu-nio-3] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.104 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.120 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:36.121 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:36.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [3ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [3ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [19ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@301188766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@20ab3d23 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14d6817b -> wakeup thread Time-limited test)) after lookup
14:22:36.122 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:36.123 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.123 [DEBUG - kudu-nio-4] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.123 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:36.140 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:36.141 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [3ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [3ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [19ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [22ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [22ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@301188766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@20ab3d23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14d6817b -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121))) after lookup
14:22:36.142 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:36.143 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.143 [DEBUG - kudu-nio-5] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.143 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:36.182 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:36.183 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [3ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [3ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [19ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [22ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [22ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [80ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@301188766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@20ab3d23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14d6817b -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458))) after lookup
14:22:36.183 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:36.184 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.185 [DEBUG - kudu-nio-6] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.185 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.221 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:36.222 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:36.222 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=120), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [3ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [3ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [19ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [22ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [22ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [80ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [82ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [84ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [84ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [120ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@301188766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@20ab3d23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458) -> (continuation of Deferred@346295141 after retry RPC@2049094232), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14d6817b -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458) -> (continuation of Deferred@346295141 after retry RPC@2049094232))) after lookup
14:22:36.223 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:36.226 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.226 [DEBUG - kudu-nio-7] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.227 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.601 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:36.602 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:36.602 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=500), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [3ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [3ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [19ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [22ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [22ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [80ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [82ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [84ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [84ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [120ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [122ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [126ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [126ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [496ms] refreshing cache from master, [496ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [500ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@301188766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@20ab3d23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458) -> (continuation of Deferred@346295141 after retry RPC@2049094232) -> (continuation of Deferred@1179176189 after retry RPC@826470495), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14d6817b -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458) -> (continuation of Deferred@346295141 after retry RPC@2049094232) -> (continuation of Deferred@1179176189 after retry RPC@826470495))) after lookup
14:22:36.603 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:36.604 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.604 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:36.605 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:37.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.395318  9213 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Starting pre-election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:37.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.395514  9213 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:37.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.396111  9213 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:37.398587  8555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:22:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:37.400111  8555 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.403453  9145 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316" is_pre_election: true
14:22:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.404523  8555 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab; no voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.404934  9213 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
14:22:37.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.468086  9218 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:37.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.468421  9218 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.470301  9218 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:37.478758  9076 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:37.480549  9076 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.483289  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.483495  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 3.
14:22:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.484031  9077 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.484519  9218 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 3 FOLLOWER]: Leader pre-election won for term 4
14:22:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.484686  9218 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 3 FOLLOWER]: Starting leader election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.484787  9218 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 3 FOLLOWER]: Advancing to term 4
14:22:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.487046  9218 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.487839  9218 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 4 election: Requested vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.488528  8622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.488745  8622 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 3 FOLLOWER]: Advancing to term 4
14:22:37.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:37.489920  9076 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.491179  8622 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 4.
14:22:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.491691  9077 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:37.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.492084  9218 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Leader election won for term 4
14:22:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.493510  9218 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 LEADER]: Becoming Leader. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Running, Role: LEADER
14:22:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.494105  9218 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.498297  8370 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 reported cstate change: term changed from 3 to 4, leader changed from df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67) to d4207c6141b143c590369fc8fef6b316 (127.8.16.66). New cstate: current_term: 4 leader_uuid: "d4207c6141b143c590369fc8fef6b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:37.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:37.881 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:37.881 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1779), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [3ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [3ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [19ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [22ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [22ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [80ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [82ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [84ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [84ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [120ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [122ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [126ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [126ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [496ms] refreshing cache from master, [496ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [500ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [502ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [504ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [504ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [1776ms] refreshing cache from master, [1776ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [1779ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@301188766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@20ab3d23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458) -> (continuation of Deferred@346295141 after retry RPC@2049094232) -> (continuation of Deferred@1179176189 after retry RPC@826470495) -> (continuation of Deferred@1862713559 after retry RPC@1755587823), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14d6817b -> wakeup thread Time-limited test -> (continuation of Deferred@1927360719 after retry RPC@1386003121) -> (continuation of Deferred@1628009544 after retry RPC@1450644458) -> (continuation of Deferred@346295141 after retry RPC@2049094232) -> (continuation of Deferred@1179176189 after retry RPC@826470495) -> (continuation of Deferred@1862713559 after retry RPC@1755587823))) after lookup
14:22:37.882 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:37.883 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] Successfully connected to peer
14:22:37.918 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa86267df, L:/127.0.0.1:36290 - R:/127.8.16.66:42745] using TOKEN/null
14:22:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.928890  8622 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Refusing update from remote peer d4207c6141b143c590369fc8fef6b316: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 4 index: 10. (index mismatch)
14:22:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.930119  9223 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
14:22:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:37.931548  9076 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 -> Peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Couldn't send request to peer df3bbabb6b104a5e8e63dca4a47eb6b1. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:37.966 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.67:42073
14:22:37.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:37.967847  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:42073
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41293
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.366014  9242 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.366354  9242 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.366432  9242 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.376140  9242 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.376598  9242 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.67
14:22:38.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.387576  9242 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:42073
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41293
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.389716  9242 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.392282  9242 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.406088  9247 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.408319  9242 server_base.cc:1061] running on GCE node
14:22:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.408420  9250 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:38.405949  9248 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.410364  9242 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:38.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.416764  9242 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.420423  9242 hybrid_clock.cc:648] HybridClock initialized: now 1779114158420301 us; error 76 us; skew 500 ppm
14:22:38.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.421260  9242 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:38.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.425462  9242 webserver.cc:533] Webserver started at http://127.8.16.67:41293/ using document root <none> and password file <none>
14:22:38.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.426599  9242 fs_manager.cc:362] Metadata directory not provided
14:22:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.426774  9242 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.435353  9242 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.000s
14:22:38.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.439110  9256 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:38.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.440917  9242 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.441215  9242 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.442137  9242 fs_report.cc:389] FS layout report
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/data
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:38.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:38.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:38.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.462416  9242 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:38.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.463543  9242 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.463900  9242 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.465106  9242 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:38.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.467751  9263 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:38.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.475049  9242 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.475169  9242 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
14:22:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.475298  9242 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:38.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.480289  9242 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:38.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.480429  9242 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
14:22:38.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.481038  9263 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap starting.
14:22:38.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.514328  9263 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Log is configured to *not* fsync() on all Append() calls
14:22:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.546375  9263 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:38.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.547327  9263 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap complete.
14:22:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.549302  9263 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent bootstrapping tablet: real 0.069s	user 0.047s	sys 0.020s
14:22:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.562093  9263 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.563524  9263 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Initialized, Role: FOLLOWER
14:22:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.563817  9242 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.67:42073
14:22:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.563885  9372 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.67:42073 every 8 connection(s)
14:22:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.564749  9263 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.565976  9242 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.566780  9263 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent starting tablet: real 0.017s	user 0.015s	sys 0.001s
14:22:38.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.573752  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 9242
14:22:38.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.594691  9376 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.594798  9374 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.595178  9376 heartbeater.cc:461] Registering TS with master...
14:22:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.595173  9374 heartbeater.cc:461] Registering TS with master...
14:22:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.595960  9375 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.596457  9375 heartbeater.cc:461] Registering TS with master...
14:22:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.596946  9376 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.597589  9375 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:38.599 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:38.600 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.66:42745
14:22:38.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.600929  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 9061
14:22:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.601094  9374 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.603101  8370 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.604754  8370 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.67:59021
14:22:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.605633  8442 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.607487  8299 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.622905  9187 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:38.738344  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 9061
14:22:38.761 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:38.761 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] handling channelInactive
14:22:38.762 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state READY due to: connection closed
14:22:38.777 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:38.779 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.779 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.779 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.800 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:38.801 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:38.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=3, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test)) after lookup
14:22:38.802 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:38.803 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.803 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.803 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.820 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:38.821 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:38.821 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=5, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [41ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [42ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [42ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [59ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477))) after lookup
14:22:38.822 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:38.823 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.823 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.823 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.841 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:38.841 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:38.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=7, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [41ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [42ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [42ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [59ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [61ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835))) after lookup
14:22:38.842 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:38.843 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.844 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.844 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:38.981 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:38.982 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=9, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=221), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [41ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [42ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [42ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [59ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [61ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [81ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [215ms] refreshing cache from master, [215ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [219ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611))) after lookup
14:22:38.983 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:38.984 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.984 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:38.985 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.421 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:39.421 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:39.422 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=11, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=660), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [41ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [42ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [42ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [59ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [61ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [81ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [215ms] refreshing cache from master, [215ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [219ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [222ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [224ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [224ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [655ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [659ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611) -> (continuation of Deferred@480574375 after retry RPC@1998900797), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611) -> (continuation of Deferred@480574375 after retry RPC@1998900797))) after lookup
14:22:39.422 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:39.424 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.424 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.424 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.522 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:39.522 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:39.522 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=13, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=761), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [41ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [42ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [42ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [59ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [61ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [81ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [215ms] refreshing cache from master, [215ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [219ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [222ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [224ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [224ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [655ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [659ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [661ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [663ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [663ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [756ms] refreshing cache from master, [756ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [760ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611) -> (continuation of Deferred@480574375 after retry RPC@1998900797) -> (continuation of Deferred@170605413 after retry RPC@1538214203), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611) -> (continuation of Deferred@480574375 after retry RPC@1998900797) -> (continuation of Deferred@170605413 after retry RPC@1538214203))) after lookup
14:22:39.523 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:39.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.524 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.524 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:39.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.609441  9376 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:39.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.938596  9390 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:39.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.938854  9390 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.940267  9390 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:39.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:39.943722  8555 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.951545  9324 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.952286  9324 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 3.
14:22:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.954000  8555 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.954578  9390 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Leader pre-election won for term 5
14:22:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.954839  9390 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Starting leader election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.954936  9390 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 4 FOLLOWER]: Advancing to term 5
14:22:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.957901  9390 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.958606  9390 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 5 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.959798  9324 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.960043  9324 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 3 FOLLOWER]: Advancing to term 5
14:22:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:39.960762  8555 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:39.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.966362  9324 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 5.
14:22:39.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.967459  8555 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:39.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.967880  9390 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Leader election won for term 5
14:22:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.968436  9390 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 LEADER]: Becoming Leader. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Running, Role: LEADER
14:22:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.969141  9390 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:39.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:39.973641  8370 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab reported cstate change: term changed from 4 to 5, leader changed from d4207c6141b143c590369fc8fef6b316 (127.8.16.66) to 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65). New cstate: current_term: 5 leader_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:40.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:40.493322  8555 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:40.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:40.505028  9324 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Refusing update from remote peer 84e8522e549f4f23b3fd7f83497bb3ab: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
14:22:40.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:40.506299  9408 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [LEADER]: Connected to new peer: Peer: permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
14:22:41.381 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:41.381 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:41.382 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=15, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2620), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [18ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [18ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [41ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [42ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [42ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [59ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [61ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [81ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [83ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [83ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [215ms] refreshing cache from master, [215ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [219ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [222ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [224ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [224ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [655ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [659ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [661ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [663ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [663ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [756ms] refreshing cache from master, [756ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [760ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [762ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [763ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [763ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2616ms] refreshing cache from master, [2616ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [2619ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@566323700(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@680bb27f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611) -> (continuation of Deferred@480574375 after retry RPC@1998900797) -> (continuation of Deferred@170605413 after retry RPC@1538214203) -> (continuation of Deferred@1215757958 after retry RPC@1159520319), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@467d5ca5 -> wakeup thread Time-limited test -> (continuation of Deferred@1638326304 after retry RPC@1293025477) -> (continuation of Deferred@319333631 after retry RPC@1321310835) -> (continuation of Deferred@127259968 after retry RPC@1162204611) -> (continuation of Deferred@480574375 after retry RPC@1998900797) -> (continuation of Deferred@170605413 after retry RPC@1538214203) -> (continuation of Deferred@1215757958 after retry RPC@1159520319))) after lookup
14:22:41.383 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:41.384 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] Successfully connected to peer
14:22:41.409 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x470444ad, L:/127.0.0.1:55772 - R:/127.8.16.65:35141] using TOKEN/null
14:22:41.445 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.66:42745
14:22:41.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.446691  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:41.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.851397  9431 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:41.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.851805  9431 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:41.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.851919  9431 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.861589  9431 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:41.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.861845  9431 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.66
14:22:41.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.873485  9431 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:41.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:41.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:41.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.875655  9431 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.878324  9431 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.892280  9437 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:41.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.892279  9436 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:41.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:41.892307  9439 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:41.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.893695  9431 server_base.cc:1061] running on GCE node
14:22:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.894868  9431 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.896248  9431 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.897490  9431 hybrid_clock.cc:648] HybridClock initialized: now 1779114161897389 us; error 93 us; skew 500 ppm
14:22:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.897861  9431 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.901060  9431 webserver.cc:533] Webserver started at http://127.8.16.66:35077/ using document root <none> and password file <none>
14:22:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.902156  9431 fs_manager.cc:362] Metadata directory not provided
14:22:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.902297  9431 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.910158  9431 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
14:22:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.914326  9445 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:41.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.916378  9431 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
14:22:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.916636  9431 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.917831  9431 fs_report.cc:389] FS layout report
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/data
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:41.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.940016  9431 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.941045  9431 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.941442  9431 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.942638  9431 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.945585  9452 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.952589  9431 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.952795  9431 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
14:22:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.952953  9431 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:41.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.958431  9431 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:41.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.958624  9431 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
14:22:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.959358  9452 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap starting.
14:22:41.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:41.985672  9452 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Log is configured to *not* fsync() on all Append() calls
14:22:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.008371  9452 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:42.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.009187  9452 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap complete.
14:22:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.011005  9452 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent bootstrapping tablet: real 0.052s	user 0.026s	sys 0.023s
14:22:42.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.019244  9452 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:42.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.020522  9452 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Initialized, Role: FOLLOWER
14:22:42.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.021539  9452 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.022899  9431 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.66:42745
14:22:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.022953  9563 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.66:42745 every 8 connection(s)
14:22:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.023756  9452 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent starting tablet: real 0.013s	user 0.011s	sys 0.002s
14:22:42.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.024541  9431 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.033627  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 9431
14:22:42.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:42.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.046466  9564 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:42.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.048511  9564 heartbeater.cc:461] Registering TS with master...
14:22:42.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.049806  9564 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:42.051 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:42.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.053498  9565 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:42.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.053664  9565 heartbeater.cc:461] Registering TS with master...
14:22:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.054230  9566 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.054482  9566 heartbeater.cc:461] Registering TS with master...
14:22:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.055052  9566 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.055356  9565 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.056789  8370 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.056938  8442 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.058671  8370 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.66:41419
14:22:42.059 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.65:35141
14:22:42.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:22:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.059968  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 8540
14:22:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.071295  8300 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.079059  8662 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:42.185276  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 8540
14:22:42.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:42.207 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] handling channelInactive
14:22:42.207 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state READY due to: connection closed
14:22:42.217 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:42.220 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.220 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.221 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.242 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:42.242 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:42.242 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [0ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [13ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [34ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@552700978(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2ce9d0b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e7d51ff -> wakeup thread Time-limited test)) after lookup
14:22:42.243 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:42.245 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.245 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.245 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:42.282 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:42.282 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [0ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [13ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [34ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [38ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@552700978(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2ce9d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e7d51ff -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214))) after lookup
14:22:42.283 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:42.284 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.284 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.285 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:42.301 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:42.302 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=94), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [0ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [13ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [34ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [38ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [75ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [78ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [78ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [93ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@552700978(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2ce9d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e7d51ff -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599))) after lookup
14:22:42.302 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:42.304 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.304 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.304 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.421 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:42.421 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:42.421 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=214), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [0ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [13ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [34ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [38ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [75ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [78ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [78ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [93ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [95ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [97ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [97ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [213ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@552700978(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2ce9d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599) -> (continuation of Deferred@1635603140 after retry RPC@783807619), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e7d51ff -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599) -> (continuation of Deferred@1635603140 after retry RPC@783807619))) after lookup
14:22:42.422 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:42.424 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.424 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:42.424 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:43.001 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:43.002 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:43.002 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=795), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [0ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [13ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [34ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [38ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [75ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [78ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [78ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [93ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [95ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [97ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [97ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [213ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [215ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [217ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [217ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [789ms] refreshing cache from master, [789ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [794ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@552700978(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2ce9d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599) -> (continuation of Deferred@1635603140 after retry RPC@783807619) -> (continuation of Deferred@435200864 after retry RPC@141709105), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e7d51ff -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599) -> (continuation of Deferred@1635603140 after retry RPC@783807619) -> (continuation of Deferred@435200864 after retry RPC@141709105))) after lookup
14:22:43.003 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:43.004 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:43.004 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:43.004 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.062448  9566 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:43.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.415339  9579 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:43.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.415597  9579 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:43.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.417593  9579 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:43.419294  9447 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:43.421429  9447 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:43.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.425685  9324 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:43.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.427213  9446 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d4207c6141b143c590369fc8fef6b316; no voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:43.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.427765  9579 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
14:22:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.546687  9584 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 84e8522e549f4f23b3fd7f83497bb3ab)
14:22:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.547375  9584 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.549532  9584 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:43.551842  9258 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:43.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:43.553941  9258 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:43.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.559263  9515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316" is_pre_election: true
14:22:43.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.559691  9515 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 4.
14:22:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.560833  9257 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab
14:22:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.561370  9584 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Leader pre-election won for term 6
14:22:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.561501  9584 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 84e8522e549f4f23b3fd7f83497bb3ab)
14:22:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.561618  9584 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 5 FOLLOWER]: Advancing to term 6
14:22:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.563781  9584 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.564740  9584 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 6 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.565353  9515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.565593  9515 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 4 FOLLOWER]: Advancing to term 6
14:22:43.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:43.566761  9258 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:43.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.570086  9515 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 6.
14:22:43.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.570708  9257 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab
14:22:43.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.571389  9584 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Leader election won for term 6
14:22:43.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.572250  9584 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 LEADER]: Becoming Leader. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Running, Role: LEADER
14:22:43.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.572947  9584 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:43.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.578322  8370 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 reported cstate change: term changed from 5 to 6, leader changed from 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65) to df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67). New cstate: current_term: 6 leader_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: HEALTHY } } }
14:22:43.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:43.802 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:43.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1595), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [0ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [13ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [34ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [38ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [75ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [78ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [78ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [93ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [95ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [97ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [97ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [213ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [215ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [217ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [217ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [789ms] refreshing cache from master, [789ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [794ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [796ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [797ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [797ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [1590ms] refreshing cache from master, [1590ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [1594ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@552700978(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2ce9d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599) -> (continuation of Deferred@1635603140 after retry RPC@783807619) -> (continuation of Deferred@435200864 after retry RPC@141709105) -> (continuation of Deferred@581549376 after retry RPC@943220195), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e7d51ff -> wakeup thread Time-limited test -> (continuation of Deferred@1844996447 after retry RPC@158220214) -> (continuation of Deferred@1875908116 after retry RPC@389744599) -> (continuation of Deferred@1635603140 after retry RPC@783807619) -> (continuation of Deferred@435200864 after retry RPC@141709105) -> (continuation of Deferred@581549376 after retry RPC@943220195))) after lookup
14:22:43.803 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:43.804 [DEBUG - kudu-nio-0] (Connection.java:757) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] Successfully connected to peer
14:22:43.823 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x58d0e803, L:/127.0.0.1:37258 - R:/127.8.16.67:42073] using TOKEN/null
14:22:43.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.829382  9515 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Refusing update from remote peer df3bbabb6b104a5e8e63dca4a47eb6b1: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 18. (index mismatch)
14:22:43.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:43.829669  9258 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Couldn't send request to peer 84e8522e549f4f23b3fd7f83497bb3ab. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:43.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.830510  9589 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
14:22:43.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.844108  9601 mvcc.cc:204] Tried to move back new op lower bound from 7287251615034810368 to 7287251614000041984. Current Snapshot: MvccSnapshot[applied={T|T < 7287251615034810368}]
14:22:43.871 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.65:35141
14:22:43.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:43.872987  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:35141
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34929
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:43.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.272151  9608 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.272491  9608 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.272570  9608 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:44.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.282657  9608 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:44.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.282975  9608 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.65
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.295032  9608 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:35141
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34929
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.296804  9608 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:44.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.298702  9608 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.309779  9614 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.309712  9613 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:44.310515  9616 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.311391  9608 server_base.cc:1061] running on GCE node
14:22:44.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.311998  9608 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.313120  9608 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:44.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.314481  9608 hybrid_clock.cc:648] HybridClock initialized: now 1779114164314413 us; error 46 us; skew 500 ppm
14:22:44.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.314832  9608 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:44.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.317749  9608 webserver.cc:533] Webserver started at http://127.8.16.65:34929/ using document root <none> and password file <none>
14:22:44.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.318519  9608 fs_manager.cc:362] Metadata directory not provided
14:22:44.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.318629  9608 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:44.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.324743  9608 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
14:22:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.328131  9622 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.329890  9608 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
14:22:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.330119  9608 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:25 on dist-test-slave-2txq"
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.330914  9608 fs_report.cc:389] FS layout report
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/data
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.345046  9608 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.346092  9608 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.346454  9608 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.347545  9608 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:44.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.350517  9629 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.356990  9608 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.357167  9608 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:22:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.357363  9608 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.360740  9608 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.360862  9608 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.000s
14:22:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.361716  9629 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap starting.
14:22:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.396210  9629 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Log is configured to *not* fsync() on all Append() calls
14:22:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.426927  9736 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.65:35141 every 8 connection(s)
14:22:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.426990  9608 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.65:35141
14:22:44.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.429065  9608 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.433007  9629 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:44.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.433990  9629 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap complete.
14:22:44.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.436054  9629 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent bootstrapping tablet: real 0.075s	user 0.055s	sys 0.016s
14:22:44.437 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.437054  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 9608
14:22:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.448139  9629 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.449787  9629 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Initialized, Role: FOLLOWER
14:22:44.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.451053  9629 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:44.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.460852  9629 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent starting tablet: real 0.025s	user 0.013s	sys 0.011s
14:22:44.481 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:44.482 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.67:42073
14:22:44.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:22:44.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.483068  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 9242
14:22:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.502003  9737 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.502467  9737 heartbeater.cc:461] Registering TS with master...
14:22:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.503651  9737 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.504904  9738 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.505076  9738 heartbeater.cc:461] Registering TS with master...
14:22:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.505841  9738 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.506417  9739 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.506595  9739 heartbeater.cc:461] Registering TS with master...
14:22:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.507234  9739 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.507512  8299 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.508689  8371 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.509398  8441 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.510703  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.65:59295
14:22:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.514017  9368 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.514514  9739 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:44.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:44.621845  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 9242
14:22:44.642 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:44.642 [DEBUG - kudu-nio-0] (Connection.java:239) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] handling channelInactive
14:22:44.642 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state READY due to: connection closed
14:22:44.643 [DEBUG - Time-limited test] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:44.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.644 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.660 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:44.661 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:44.661 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [18ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1729579422(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@b695dd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fcd3297 -> wakeup thread Time-limited test)) after lookup
14:22:44.662 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:44.663 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.663 [DEBUG - kudu-nio-2] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.663 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.681 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:44.682 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:44.682 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [18ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [21ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [21ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1729579422(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@b695dd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fcd3297 -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708))) after lookup
14:22:44.683 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:44.684 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.684 [DEBUG - kudu-nio-3] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.684 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.700 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:44.700 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:44.700 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [18ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [21ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [21ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [57ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1729579422(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@b695dd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fcd3297 -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897))) after lookup
14:22:44.701 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:44.703 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.703 [DEBUG - kudu-nio-4] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.703 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.721 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:44.722 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:44.722 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [18ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [21ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [21ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [57ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [59ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [61ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [61ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1729579422(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@b695dd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897) -> (continuation of Deferred@610310888 after retry RPC@1608635983), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fcd3297 -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897) -> (continuation of Deferred@610310888 after retry RPC@1608635983))) after lookup
14:22:44.723 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:44.724 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.725 [DEBUG - kudu-nio-5] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.725 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.760 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:44.760 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:44.760 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=118), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [18ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [21ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [21ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [57ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [59ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [61ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [61ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [81ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [83ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [83ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [114ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [117ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1729579422(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@b695dd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897) -> (continuation of Deferred@610310888 after retry RPC@1608635983) -> (continuation of Deferred@1128060303 after retry RPC@314454025), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fcd3297 -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897) -> (continuation of Deferred@610310888 after retry RPC@1608635983) -> (continuation of Deferred@1128060303 after retry RPC@314454025))) after lookup
14:22:44.761 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:44.763 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.763 [DEBUG - kudu-nio-6] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:44.763 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:45.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.751959  9752 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:45.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.752244  9752 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:45.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.754211  9752 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:45.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:45.756040  9623 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:45.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:45.757725  9623 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:45.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.761678  9515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316" is_pre_election: true
14:22:45.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.762668  9623 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab; no voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:45.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.763146  9752 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
14:22:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.783777  9757 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.784031  9757 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.784824  9757 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:45.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:45.787607  9446 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:45.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.793637  9691 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:45.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.794080  9691 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 5.
14:22:45.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.795179  9447 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:45.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.795529  9757 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Leader pre-election won for term 7
14:22:45.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.795662  9757 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Starting leader election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:45.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.795735  9757 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 6 FOLLOWER]: Advancing to term 7
14:22:45.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.797946  9757 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:45.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.798631  9757 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 7 election: Requested vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.799290  9691 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.799561  9691 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 5 FOLLOWER]: Advancing to term 7
14:22:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:45.800384  9446 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.803835  9691 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 7.
14:22:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.804527  9447 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.805020  9757 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Leader election won for term 7
14:22:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.805564  9757 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 LEADER]: Becoming Leader. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Running, Role: LEADER
14:22:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.806180  9757 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:45.810000  8371 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 reported cstate change: term changed from 6 to 7, leader changed from df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67) to d4207c6141b143c590369fc8fef6b316 (127.8.16.66). New cstate: current_term: 7 leader_uuid: "d4207c6141b143c590369fc8fef6b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:46.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:46.181 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:46.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1539), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [18ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [20ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [21ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [21ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [39ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [40ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [42ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [42ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [57ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [59ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [61ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [61ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [79ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [81ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [83ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [83ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [114ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [117ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [119ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [121ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [121ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [1535ms] refreshing cache from master, [1535ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [1538ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1729579422(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@b695dd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897) -> (continuation of Deferred@610310888 after retry RPC@1608635983) -> (continuation of Deferred@1128060303 after retry RPC@314454025) -> (continuation of Deferred@981192878 after retry RPC@1504084368), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fcd3297 -> wakeup thread Time-limited test -> (continuation of Deferred@415908072 after retry RPC@235241708) -> (continuation of Deferred@781061460 after retry RPC@212789897) -> (continuation of Deferred@610310888 after retry RPC@1608635983) -> (continuation of Deferred@1128060303 after retry RPC@314454025) -> (continuation of Deferred@981192878 after retry RPC@1504084368))) after lookup
14:22:46.182 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:46.183 [DEBUG - kudu-nio-7] (Connection.java:757) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] Successfully connected to peer
14:22:46.208 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x89c0b235, L:/127.0.0.1:49960 - R:/127.8.16.66:42745] using TOKEN/null
14:22:46.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.211889  9446 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 -> Peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Couldn't send request to peer df3bbabb6b104a5e8e63dca4a47eb6b1. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.215339  9691 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Refusing update from remote peer d4207c6141b143c590369fc8fef6b316: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 22. (index mismatch)
14:22:46.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.216502  9757 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
14:22:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.230727  9775 mvcc.cc:204] Tried to move back new op lower bound from 7287251624807473152 to 7287251623146450944. Current Snapshot: MvccSnapshot[applied={T|T < 7287251624807473152}]
14:22:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.236956  9776 mvcc.cc:204] Tried to move back new op lower bound from 7287251624807473152 to 7287251623146450944. Current Snapshot: MvccSnapshot[applied={T|T < 7287251624807473152 or (T in {7287251624807473152})}]
14:22:46.264 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.67:42073
14:22:46.265 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:22:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.266618  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:42073
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41293
14:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.647112  9782 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.647437  9782 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.647516  9782 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:46.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.656586  9782 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:46.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.656781  9782 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.67
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.667075  9782 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:42073
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41293
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.668685  9782 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:46.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.670707  9782 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.686311  9787 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.686432  9788 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:46.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:46.687417  9790 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.688571  9782 server_base.cc:1061] running on GCE node
14:22:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.689339  9782 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.690362  9782 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.691609  9782 hybrid_clock.cc:648] HybridClock initialized: now 1779114166691571 us; error 31 us; skew 500 ppm
14:22:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.691938  9782 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.694702  9782 webserver.cc:533] Webserver started at http://127.8.16.67:41293/ using document root <none> and password file <none>
14:22:46.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.695530  9782 fs_manager.cc:362] Metadata directory not provided
14:22:46.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.695639  9782 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.702090  9782 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
14:22:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.705665  9796 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:46.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.707558  9782 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.707768  9782 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.708565  9782 fs_report.cc:389] FS layout report
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/data
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:46.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:46.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.723656  9782 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:46.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.724584  9782 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:46.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.724852  9782 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:46.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.726130  9782 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:46.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.728621  9803 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:46.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.734972  9782 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:46.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.735093  9782 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:22:46.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.735232  9782 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:46.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.739024  9782 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:46.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.739192  9782 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
14:22:46.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.739781  9803 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap starting.
14:22:46.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.767665  9803 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Log is configured to *not* fsync() on all Append() calls
14:22:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.800336  9803 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.801339  9803 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap complete.
14:22:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.801918  9782 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.67:42073
14:22:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.802044  9910 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.67:42073 every 8 connection(s)
14:22:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.803090  9803 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent bootstrapping tablet: real 0.064s	user 0.037s	sys 0.024s
14:22:46.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.804118  9782 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.809348  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 9782
14:22:46.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:46.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.811880  9803 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:46.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.813407  9803 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Initialized, Role: FOLLOWER
14:22:46.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.814509  9803 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.825047  9803 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent starting tablet: real 0.022s	user 0.013s	sys 0.008s
14:22:46.836 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:46.842 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.66:42745
14:22:46.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:46.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.842913  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 9431
14:22:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.872655  9912 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.873364  9912 heartbeater.cc:461] Registering TS with master...
14:22:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.874680  9912 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:46.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.879292  8441 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:46.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.879513  9558 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:46.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.886719  9911 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.886911  9911 heartbeater.cc:461] Registering TS with master...
14:22:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.887454  9911 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:46.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.889106  8299 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:46.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.891686  9913 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:46.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.891857  9913 heartbeater.cc:461] Registering TS with master...
14:22:46.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.892448  9913 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.893829  8371 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.895152  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.67:40943
14:22:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:46.984365  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 9431
14:22:47.004 [DEBUG - kudu-nio-7] (Connection.java:239) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] handling channelInactive
14:22:47.004 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state READY due to: connection closed
14:22:47.004 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:47.005 [DEBUG - Time-limited test] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:47.006 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.007 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.007 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.020 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:47.020 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:47.020 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [14ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@262755421(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@325ab141 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@398d281f -> wakeup thread Time-limited test)) after lookup
14:22:47.021 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:47.022 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.022 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.022 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:47.040 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:47.041 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [14ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [16ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [17ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [17ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [35ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@262755421(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@325ab141 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@398d281f -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419))) after lookup
14:22:47.042 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:47.043 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.043 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.044 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:47.081 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:47.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [14ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [16ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [17ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [17ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [35ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [39ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [39ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [75ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@262755421(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@325ab141 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@398d281f -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056))) after lookup
14:22:47.082 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:47.083 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.083 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.083 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:47.121 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:47.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [14ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [16ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [17ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [17ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [35ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [39ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [39ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [75ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [77ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [78ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [78ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [115ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@262755421(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@325ab141 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056) -> (continuation of Deferred@953203059 after retry RPC@1630250413), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@398d281f -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056) -> (continuation of Deferred@953203059 after retry RPC@1630250413))) after lookup
14:22:47.122 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:47.124 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.124 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.124 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.560 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:47.560 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:47.561 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=556), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [14ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [16ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [17ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [17ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [35ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [39ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [39ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [75ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [77ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [78ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [78ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [115ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [117ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [119ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [119ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [555ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@262755421(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@325ab141 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056) -> (continuation of Deferred@953203059 after retry RPC@1630250413) -> (continuation of Deferred@1068038674 after retry RPC@59913863), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@398d281f -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056) -> (continuation of Deferred@953203059 after retry RPC@1630250413) -> (continuation of Deferred@1068038674 after retry RPC@59913863))) after lookup
14:22:47.562 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:47.563 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.563 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.563 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:47.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:47.898651  9913 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:48.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.018193  9926 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:48.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.018455  9926 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:48.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.020181  9926 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:48.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:48.021553  9797 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:48.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:48.023289  9797 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.027397  9691 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.028460  9798 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316
14:22:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.028889  9926 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
14:22:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.151772  9931 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Starting pre-election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.152012  9931 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:48.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.152832  9931 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:48.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:48.156320  9623 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.161928  9866 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.162379  9866 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 6.
14:22:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.163417  9623 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:48.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.163827  9931 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Leader pre-election won for term 8
14:22:48.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.163956  9931 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Starting leader election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:48.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.164023  9931 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 7 FOLLOWER]: Advancing to term 8
14:22:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.166239  9931 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.167047  9931 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 8 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.168077  9866 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.168292  9866 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 6 FOLLOWER]: Advancing to term 8
14:22:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:48.168565  9623 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.172423  9866 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 8.
14:22:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.173024  9623 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.173388  9931 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Leader election won for term 8
14:22:48.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.174216  9931 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 LEADER]: Becoming Leader. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Running, Role: LEADER
14:22:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.174836  9931 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.178856  8371 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab reported cstate change: term changed from 7 to 8, leader changed from d4207c6141b143c590369fc8fef6b316 (127.8.16.66) to 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65). New cstate: current_term: 8 leader_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:48.579499  9623 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.593072  9866 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Refusing update from remote peer 84e8522e549f4f23b3fd7f83497bb3ab: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
14:22:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:48.594251  9931 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [LEADER]: Connected to new peer: Peer: permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
14:22:49.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:49.320 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:49.321 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2315), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [14ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [16ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [17ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [17ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [35ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [36ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [39ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [39ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [75ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [77ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [78ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [78ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [115ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [117ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [119ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [119ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [555ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [556ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [558ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [558ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2311ms] refreshing cache from master, [2311ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [2314ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@262755421(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@325ab141 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056) -> (continuation of Deferred@953203059 after retry RPC@1630250413) -> (continuation of Deferred@1068038674 after retry RPC@59913863) -> (continuation of Deferred@1452768932 after retry RPC@1143164159), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@398d281f -> wakeup thread Time-limited test -> (continuation of Deferred@1081348594 after retry RPC@283706419) -> (continuation of Deferred@1087456213 after retry RPC@406921056) -> (continuation of Deferred@953203059 after retry RPC@1630250413) -> (continuation of Deferred@1068038674 after retry RPC@59913863) -> (continuation of Deferred@1452768932 after retry RPC@1143164159))) after lookup
14:22:49.321 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:49.322 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] Successfully connected to peer
14:22:49.348 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x2ce60481, L:/127.0.0.1:55830 - R:/127.8.16.65:35141] using TOKEN/null
14:22:49.380 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.66:42745
14:22:49.380 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.381706  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:49.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.759608  9965 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:49.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.759969  9965 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:49.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.760048  9965 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.768687  9965 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.768874  9965 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.66
14:22:49.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.779565  9965 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.781615  9965 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.784126  9965 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.799032  9973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.799079  9972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:49.799765  9975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.800165  9965 server_base.cc:1061] running on GCE node
14:22:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.800812  9965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.802078  9965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.803321  9965 hybrid_clock.cc:648] HybridClock initialized: now 1779114169803255 us; error 55 us; skew 500 ppm
14:22:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.803699  9965 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:49.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.806555  9965 webserver.cc:533] Webserver started at http://127.8.16.66:35077/ using document root <none> and password file <none>
14:22:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.807559  9965 fs_manager.cc:362] Metadata directory not provided
14:22:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.807678  9965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.815435  9965 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.003s
14:22:49.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.819288  9981 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.821101  9965 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.000s
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.821435  9965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.822194  9965 fs_report.cc:389] FS layout report
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/data
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:49.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.846565  9965 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.847460  9965 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.847826  9965 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.849108  9965 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.851727  9988 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.858703  9965 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.858829  9965 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
14:22:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.858963  9965 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.862489  9965 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.862607  9965 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
14:22:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.863205  9988 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap starting.
14:22:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.892833  9988 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Log is configured to *not* fsync() on all Append() calls
14:22:49.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.917603  9965 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.66:42745
14:22:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.917639 10095 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.66:42745 every 8 connection(s)
14:22:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.919252  9965 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:22:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.926020  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 9965
14:22:49.927 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.935230  9988 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:49.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.936417  9988 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap complete.
14:22:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.938035  9988 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent bootstrapping tablet: real 0.075s	user 0.053s	sys 0.020s
14:22:49.942 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:49.943 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.65:35141
14:22:49.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:22:49.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.944139  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 9608
14:22:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.951932  9988 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.953456  9988 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Initialized, Role: FOLLOWER
14:22:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.958237  9988 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:49.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.982448  9988 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent starting tablet: real 0.044s	user 0.015s	sys 0.015s
14:22:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.996245 10100 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.996703 10100 heartbeater.cc:461] Registering TS with master...
14:22:49.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.998093 10100 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:49.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:49.998733  9732 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:50.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.002349  8371 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.004477  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.66:50449
14:22:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.008157 10100 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.009428 10096 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.009606 10096 heartbeater.cc:461] Registering TS with master...
14:22:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.010010 10096 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.010141 10097 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.010293 10097 heartbeater.cc:461] Registering TS with master...
14:22:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.010596 10097 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.012441  8441 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.012518  8300 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:22:50.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:50.111493  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 9608
14:22:50.134 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] handling channelInactive
14:22:50.135 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state READY due to: connection closed
14:22:50.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:50.135612 10111 negotiation.cc:336] Failed RPC negotiation. Trace:
14:22:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.111677 (+     0us) reactor.cc:678] Submitting negotiation task for server connection from 127.8.16.65:35449 (local address 127.8.16.66:42745)
14:22:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.117730 (+  6053us) server_negotiation.cc:207] Beginning negotiation
14:22:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.117738 (+     8us) server_negotiation.cc:400] Waiting for connection header
14:22:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.117773 (+    35us) server_negotiation.cc:408] Connection header received
14:22:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.117867 (+    94us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:22:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.117874 (+     7us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:22:50.136 [DEBUG - Time-limited test] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.118108 (+   234us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:22:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.118309 (+   201us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:22:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:50.134336 (+ 16027us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.8.16.65:35449: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:22:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5908,"thread_start_us":5736,"threads_started":1}
14:22:50.137 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.137 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.137 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.159 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:50.160 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:50.160 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [24ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@261919185(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@51866069 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@692bae0d -> wakeup thread Time-limited test)) after lookup
14:22:50.161 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:50.162 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.162 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.162 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:50.180 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:50.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [24ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [25ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [27ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [27ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [45ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@261919185(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@51866069 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@692bae0d -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482))) after lookup
14:22:50.181 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:50.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.182 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.222 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:50.223 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:50.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [24ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [25ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [27ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [27ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [45ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [45ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [47ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [47ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [87ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@261919185(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@51866069 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@692bae0d -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731))) after lookup
14:22:50.224 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:50.225 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.226 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.226 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:50.281 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:50.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=146), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [24ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [25ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [27ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [27ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [45ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [45ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [47ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [47ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [87ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [89ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [91ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [91ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [145ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@261919185(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@51866069 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731) -> (continuation of Deferred@1390653021 after retry RPC@2100802840), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@692bae0d -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731) -> (continuation of Deferred@1390653021 after retry RPC@2100802840))) after lookup
14:22:50.282 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:50.284 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.284 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.284 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:50.481 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:50.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=346), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [24ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [25ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [27ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [27ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [45ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [45ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [47ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [47ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [87ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [89ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [91ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [91ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [145ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [147ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [149ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [149ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [342ms] refreshing cache from master, [342ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [345ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@261919185(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@51866069 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731) -> (continuation of Deferred@1390653021 after retry RPC@2100802840) -> (continuation of Deferred@850734690 after retry RPC@1927991897), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@692bae0d -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731) -> (continuation of Deferred@1390653021 after retry RPC@2100802840) -> (continuation of Deferred@850734690 after retry RPC@1927991897))) after lookup
14:22:50.482 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:22:50.483 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.483 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:50.483 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:22:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.116720 10112 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.117172 10112 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:51.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.119107 10112 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:51.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:51.120434  9983 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:51.122745  9983 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.127343  9866 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.128350  9982 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d4207c6141b143c590369fc8fef6b316; no voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.128867 10112 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
14:22:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.332226 10117 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 84e8522e549f4f23b3fd7f83497bb3ab)
14:22:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.332450 10117 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.333284 10117 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:51.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:51.345893  9798 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.353498 10051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316" is_pre_election: true
14:22:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.354058 10051 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 7.
14:22:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.355225  9797 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab
14:22:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.355651 10117 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Leader pre-election won for term 9
14:22:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.355818 10117 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 84e8522e549f4f23b3fd7f83497bb3ab)
14:22:51.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.355937 10117 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 8 FOLLOWER]: Advancing to term 9
14:22:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.360126 10117 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.361160 10117 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 9 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.361866 10051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316"
14:22:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.362138 10051 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 7 FOLLOWER]: Advancing to term 9
14:22:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:51.363016  9798 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:22:51.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.368884 10051 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 9.
14:22:51.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.369689  9797 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab
14:22:51.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.370179 10117 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Leader election won for term 9
14:22:51.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.370760 10117 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 LEADER]: Becoming Leader. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Running, Role: LEADER
14:22:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.371361 10117 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.377111  8371 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 reported cstate change: term changed from 8 to 9, leader changed from 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65) to df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67). New cstate: current_term: 9 leader_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: HEALTHY } } }
14:22:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:51.781512  9798 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Couldn't send request to peer 84e8522e549f4f23b3fd7f83497bb3ab. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.808868 10051 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Refusing update from remote peer df3bbabb6b104a5e8e63dca4a47eb6b1: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
14:22:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:51.810106 10117 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
14:22:52.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:52.441 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:52.442 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2306), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [24ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [25ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [27ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [27ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [45ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [45ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [47ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [47ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [87ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [89ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [91ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [91ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [145ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [147ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [149ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [149ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [342ms] refreshing cache from master, [342ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [345ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [347ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [348ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [348ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2302ms] refreshing cache from master, [2302ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [2305ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@261919185(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@51866069 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731) -> (continuation of Deferred@1390653021 after retry RPC@2100802840) -> (continuation of Deferred@850734690 after retry RPC@1927991897) -> (continuation of Deferred@292590036 after retry RPC@892137913), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@692bae0d -> wakeup thread Time-limited test -> (continuation of Deferred@1603579170 after retry RPC@137349482) -> (continuation of Deferred@981370959 after retry RPC@1282175731) -> (continuation of Deferred@1390653021 after retry RPC@2100802840) -> (continuation of Deferred@850734690 after retry RPC@1927991897) -> (continuation of Deferred@292590036 after retry RPC@892137913))) after lookup
14:22:52.442 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:52.443 [DEBUG - kudu-nio-5] (Connection.java:757) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] Successfully connected to peer
14:22:52.472 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x76650610, L:/127.0.0.1:45072 - R:/127.8.16.67:42073] using TOKEN/null
14:22:52.498 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.65:35141
14:22:52.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.499969  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:35141
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34929
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:52.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.868470 10150 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:52.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.868801 10150 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:52.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.868880 10150 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:52.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.877696 10150 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:52.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.877943 10150 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.65
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.888224 10150 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:35141
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34929
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:52.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.889904 10150 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.891710 10150 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.901973 10158 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.902011 10156 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:52.901923 10155 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.903406 10150 server_base.cc:1061] running on GCE node
14:22:52.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.904124 10150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:52.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.905313 10150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:52.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.906584 10150 hybrid_clock.cc:648] HybridClock initialized: now 1779114172906511 us; error 60 us; skew 500 ppm
14:22:52.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.907007 10150 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:52.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.909870 10150 webserver.cc:533] Webserver started at http://127.8.16.65:34929/ using document root <none> and password file <none>
14:22:52.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.910706 10150 fs_manager.cc:362] Metadata directory not provided
14:22:52.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.910848 10150 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.917218 10150 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
14:22:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.921053 10164 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.922972 10150 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.923396 10150 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:25 on dist-test-slave-2txq"
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.924192 10150 fs_report.cc:389] FS layout report
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/data
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:52.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.946624 10150 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.947738 10150 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:52.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.948112 10150 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.949256 10150 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.951799 10171 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.960530 10150 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.960769 10150 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.000s	sys 0.001s
14:22:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.960913 10150 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:52.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.964164 10150 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:52.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.964313 10150 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
14:22:52.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:52.964799 10171 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap starting.
14:22:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.001797 10171 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Log is configured to *not* fsync() on all Append() calls
14:22:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.022850 10150 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.65:35141
14:22:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.022919 10278 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.65:35141 every 8 connection(s)
14:22:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.026047 10150 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:22:53.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.032593  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 10150
14:22:53.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:53.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.052135 10171 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.053573 10171 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap complete.
14:22:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.055577 10171 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent bootstrapping tablet: real 0.091s	user 0.069s	sys 0.017s
14:22:53.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.063608 10171 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:53.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.064975 10171 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Initialized, Role: FOLLOWER
14:22:53.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.065951 10171 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:53.107 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:53.108 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.67:42073
14:22:53.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:22:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.108634  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 9782
14:22:53.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.119711 10281 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:53.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.120236 10281 heartbeater.cc:461] Registering TS with master...
14:22:53.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.121562 10281 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.124794 10280 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.125032 10280 heartbeater.cc:461] Registering TS with master...
14:22:53.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.125872 10280 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.128482  8371 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.130434  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.65:56285
14:22:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.133920  8441 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:53.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.145483 10171 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent starting tablet: real 0.090s	user 0.036s	sys 0.054s
14:22:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.148087 10279 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.148227 10279 heartbeater.cc:461] Registering TS with master...
14:22:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.148527 10279 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.149456 10281 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.150717  8300 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.162261  9906 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:53.265597  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 9782
14:22:53.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:53.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:53.287705 10296 negotiation.cc:336] Failed RPC negotiation. Trace:
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.127131 (+     0us) reactor.cc:678] Submitting negotiation task for server connection from 127.8.16.67:55911 (local address 127.8.16.65:35141)
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.157918 (+ 30787us) server_negotiation.cc:207] Beginning negotiation
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.157932 (+    14us) server_negotiation.cc:400] Waiting for connection header
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.159462 (+  1530us) server_negotiation.cc:408] Connection header received
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.159577 (+   115us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.159587 (+    10us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.159929 (+   342us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.160154 (+   225us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.161596 (+  1442us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.164303 (+  2707us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:22:53.286413 (+122110us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.8.16.67:55911: BlockingRecv error: recv got EOF from 127.8.16.67:55911 (error 108)
14:22:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":30557,"thread_start_us":30827,"threads_started":1}
14:22:53.289 [DEBUG - kudu-nio-5] (Connection.java:239) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] handling channelInactive
14:22:53.289 [DEBUG - kudu-nio-5] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state READY due to: connection closed
14:22:53.289 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: connection closed
14:22:53.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:53.300 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:53.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [13ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@2056679838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2865f64c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@186a3d36 -> wakeup thread Time-limited test)) after lookup
14:22:53.301 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:53.302 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.302 [DEBUG - kudu-nio-6] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.302 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:53.320 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:53.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [13ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [14ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [15ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [33ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@2056679838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2865f64c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@186a3d36 -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713))) after lookup
14:22:53.321 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:53.322 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.323 [DEBUG - kudu-nio-7] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.323 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.360 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:53.360 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:53.360 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [13ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [14ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [15ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [33ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [34ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [36ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [36ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@2056679838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2865f64c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@186a3d36 -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056))) after lookup
14:22:53.361 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:53.362 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.362 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.362 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:53.439 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:53.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=152), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [13ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [14ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [15ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [33ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [34ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [36ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [36ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [74ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [75ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [75ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [152ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@2056679838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2865f64c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056) -> (continuation of Deferred@1490511530 after retry RPC@516589605), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@186a3d36 -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056) -> (continuation of Deferred@1490511530 after retry RPC@516589605))) after lookup
14:22:53.440 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:53.441 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.441 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.821 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:53.821 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:22:53.822 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=534), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [13ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [14ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [15ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [33ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [34ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [36ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [36ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [74ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [75ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [75ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [152ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [153ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [154ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [154ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [529ms] refreshing cache from master, [529ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [533ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@2056679838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2865f64c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056) -> (continuation of Deferred@1490511530 after retry RPC@516589605) -> (continuation of Deferred@2019432905 after retry RPC@840904007), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@186a3d36 -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056) -> (continuation of Deferred@1490511530 after retry RPC@516589605) -> (continuation of Deferred@2019432905 after retry RPC@840904007))) after lookup
14:22:53.822 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:22:53.823 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.824 [DEBUG - kudu-nio-2] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state CONNECTING due to: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:53.824 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073
14:22:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.400110 10298 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.400427 10298 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.402395 10298 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:54.404305 10165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:54.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:54.406680 10165 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.409497 10051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316" is_pre_election: true
14:22:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.410602 10165 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab; no voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.411137 10298 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
14:22:54.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.575795 10303 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:54.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.576092 10303 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.577055 10303 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:54.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:54.579952  9982 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.586316 10233 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.586802 10233 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 8.
14:22:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.588462  9983 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.588812 10303 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Leader pre-election won for term 10
14:22:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.588959 10303 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Starting leader election (detected failure of leader df3bbabb6b104a5e8e63dca4a47eb6b1)
14:22:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.589033 10303 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 9 FOLLOWER]: Advancing to term 10
14:22:54.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.591728 10303 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:54.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.592458 10303 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 10 election: Requested vote from peers 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:54.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.593036 10233 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "d4207c6141b143c590369fc8fef6b316" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:22:54.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.593302 10233 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 8 FOLLOWER]: Advancing to term 10
14:22:54.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:54.594812  9982 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111)
14:22:54.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.598841 10233 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d4207c6141b143c590369fc8fef6b316 in term 10.
14:22:54.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.599926  9983 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316; no voters: df3bbabb6b104a5e8e63dca4a47eb6b1
14:22:54.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.600659 10303 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 FOLLOWER]: Leader election won for term 10
14:22:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.601550 10303 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 LEADER]: Becoming Leader. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Running, Role: LEADER
14:22:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.602296 10303 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:54.607065  8371 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 reported cstate change: term changed from 9 to 10, leader changed from df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67) to d4207c6141b143c590369fc8fef6b316 (127.8.16.66). New cstate: current_term: 10 leader_uuid: "d4207c6141b143c590369fc8fef6b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:54.986881  9982 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 -> Peer df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073): Couldn't send request to peer df3bbabb6b104a5e8e63dca4a47eb6b1. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.67:42073: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:55.211133 10233 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Refusing update from remote peer d4207c6141b143c590369fc8fef6b316: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 33. (index mismatch)
14:22:55.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:55.212170 10303 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
14:22:55.620 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:55.621 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:55.621 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2333), Traces: [0ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [2ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [13ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [14ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [15ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [15ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [33ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [34ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [36ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [36ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [73ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [74ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [75ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [75ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [152ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [153ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [154ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [154ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [529ms] refreshing cache from master, [529ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [533ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [535ms] sending RPC to server df3bbabb6b104a5e8e63dca4a47eb6b1, [537ms] received response from server df3bbabb6b104a5e8e63dca4a47eb6b1: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [537ms] delaying RPC due to: Network error: Failed to connect to peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073): Connection refused: /127.8.16.67:42073, [2330ms] refreshing cache from master, [2330ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [2333ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@2056679838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@2865f64c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056) -> (continuation of Deferred@1490511530 after retry RPC@516589605) -> (continuation of Deferred@2019432905 after retry RPC@840904007) -> (continuation of Deferred@84057727 after retry RPC@1321008781), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@186a3d36 -> wakeup thread Time-limited test -> (continuation of Deferred@1418338243 after retry RPC@490740713) -> (continuation of Deferred@1291092492 after retry RPC@1366710056) -> (continuation of Deferred@1490511530 after retry RPC@516589605) -> (continuation of Deferred@2019432905 after retry RPC@840904007) -> (continuation of Deferred@84057727 after retry RPC@1321008781))) after lookup
14:22:55.622 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:55.623 [DEBUG - kudu-nio-3] (Connection.java:757) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] Successfully connected to peer
14:22:55.641 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x366f0502, L:/127.0.0.1:46638 - R:/127.8.16.66:42745] using TOKEN/null
14:22:55.670 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.67:42073
14:22:55.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:55.671962  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:42073
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41293
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:22:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.065701 10332 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:22:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.066049 10332 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.066126 10332 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:22:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.076740 10332 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:22:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.076960 10332 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.67
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.088234 10332 tablet_server_runner.cc:78] Tablet server non-default flags:
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.67:42073
14:22:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.67
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41293
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.67
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/logs
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:22:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:22:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.089968 10332 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.091869 10332 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:56.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.104202 10337 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.104137 10338 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:56.104877 10340 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.105346 10332 server_base.cc:1061] running on GCE node
14:22:56.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.105939 10332 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.107060 10332 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.108474 10332 hybrid_clock.cc:648] HybridClock initialized: now 1779114176108390 us; error 60 us; skew 500 ppm
14:22:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.108915 10332 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.111624 10332 webserver.cc:533] Webserver started at http://127.8.16.67:41293/ using document root <none> and password file <none>
14:22:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.112534 10332 fs_manager.cc:362] Metadata directory not provided
14:22:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.112691 10332 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.119329 10332 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
14:22:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.123342 10346 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:22:56.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.126009 10332 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.001s	sys 0.004s
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.126252 10332 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.127135 10332 fs_report.cc:389] FS layout report
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/wal
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/data
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.150825 10332 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:22:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.151782 10332 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.152093 10332 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.153812 10332 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:22:56.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.156599 10353 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:22:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.163447 10332 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:22:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.163571 10332 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:22:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.163683 10332 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:22:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.167282 10332 ts_tablet_manager.cc:616] Registered 1 tablets
14:22:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.167428 10332 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
14:22:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.168474 10353 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap starting.
14:22:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.212090 10353 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Log is configured to *not* fsync() on all Append() calls
14:22:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.231101 10332 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.67:42073
14:22:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.231192 10460 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.67:42073 every 8 connection(s)
14:22:56.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.234503 10332 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-2/data/info.pb
14:22:56.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.237512  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 10332
14:22:56.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:56.252 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:22:56.253 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.66:42745
14:22:56.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:22:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.254168  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 9965
14:22:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.302603 10462 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:22:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.303110 10462 heartbeater.cc:461] Registering TS with master...
14:22:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.303158 10353 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:22:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.304318 10461 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:22:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.305577 10353 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Bootstrap complete.
14:22:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.306239 10091 generic_service.cc:196] Checking for leaks (request via RPC)
14:22:56.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.310015 10461 heartbeater.cc:461] Registering TS with master...
14:22:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.317330 10462 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:22:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.317442 10461 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:22:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.320290 10353 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent bootstrapping tablet: real 0.153s	user 0.078s	sys 0.035s
14:22:56.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.324718  8299 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:56.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.328445  8442 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.329032 10468 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.329262 10468 heartbeater.cc:461] Registering TS with master...
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.329929 10468 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.331415  8370 ts_manager.cc:194] Re-registered known tserver with Master: df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.331157 10353 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.332535 10353 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Initialized, Role: FOLLOWER
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.333339 10353 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.334626  8370 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.67:37131
14:22:56.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.337157 10353 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1: Time spent starting tablet: real 0.017s	user 0.009s	sys 0.004s
14:22:56.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.338578 10468 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:22:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:56.422078  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 9965
14:22:56.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:22:56.442 [DEBUG - kudu-nio-3] (Connection.java:239) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] handling channelInactive
14:22:56.442 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state READY due to: connection closed
14:22:56.443 [DEBUG - Time-limited test] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:56.444 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.444 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.444 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.459 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:56.460 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:56.461 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test)) after lookup
14:22:56.462 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:56.463 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.463 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.463 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.480 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:56.480 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:56.480 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [21ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [21ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070))) after lookup
14:22:56.481 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:56.482 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.482 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.483 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:56.501 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:56.502 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [21ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [21ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [39ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [40ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [58ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358))) after lookup
14:22:56.502 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:56.503 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.504 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.504 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.621 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:56.621 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:56.621 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=179), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [21ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [21ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [39ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [40ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [58ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [60ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [178ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610))) after lookup
14:22:56.622 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:56.623 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.623 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.623 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.760 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:56.761 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:56.761 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=318), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [21ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [21ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [39ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [40ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [58ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [60ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [178ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [179ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [181ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [181ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [315ms] refreshing cache from master, [315ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [318ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610) -> (continuation of Deferred@1502706929 after retry RPC@1760246875), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610) -> (continuation of Deferred@1502706929 after retry RPC@1760246875))) after lookup
14:22:56.762 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:56.763 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.763 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:56.763 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:57.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:22:57.440 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)[L],df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:22:57.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=998), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [21ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [21ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [39ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [40ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [58ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [60ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [178ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [179ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [181ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [181ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [315ms] refreshing cache from master, [315ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [318ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [320ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [321ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [321ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [994ms] refreshing cache from master, [994ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [998ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610) -> (continuation of Deferred@1502706929 after retry RPC@1760246875) -> (continuation of Deferred@24325185 after retry RPC@810275280), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610) -> (continuation of Deferred@1502706929 after retry RPC@1760246875) -> (continuation of Deferred@24325185 after retry RPC@810275280))) after lookup
14:22:57.442 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] connecting to peer
14:22:57.443 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:57.443 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] cleaning up while in state CONNECTING due to: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:57.443 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745
14:22:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.459506 10476 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:22:57.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.459802 10476 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.461592 10476 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:22:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:57.463219 10347 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:57.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:57.465318 10347 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:57.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.470026 10233 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" is_pre_election: true
14:22:57.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.471163 10348 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab, d4207c6141b143c590369fc8fef6b316
14:22:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.471634 10476 raft_consensus.cc:2749] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
14:22:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.791755 10481 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Starting pre-election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.792089 10481 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:57.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.792922 10481 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:57.795795 10165 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:57.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.802369 10415 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" is_pre_election: true
14:22:57.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.802742 10415 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 9.
14:22:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.803787 10165 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.804243 10481 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Leader pre-election won for term 11
14:22:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.804394 10481 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Starting leader election (detected failure of leader d4207c6141b143c590369fc8fef6b316)
14:22:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.804471 10481 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 10 FOLLOWER]: Advancing to term 11
14:22:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.806753 10481 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.807355 10481 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 11 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67:42073)
14:22:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.808492 10415 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1"
14:22:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.808780 10415 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 9 FOLLOWER]: Advancing to term 11
14:22:57.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:57.809391 10165 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111)
14:22:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.812860 10415 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84e8522e549f4f23b3fd7f83497bb3ab in term 11.
14:22:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.813527 10165 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 84e8522e549f4f23b3fd7f83497bb3ab, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: d4207c6141b143c590369fc8fef6b316
14:22:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.813954 10481 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 11 FOLLOWER]: Leader election won for term 11
14:22:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.814405 10481 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 11 LEADER]: Becoming Leader. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Running, Role: LEADER
14:22:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.814924 10481 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:22:57.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:57.819078  8370 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab reported cstate change: term changed from 10 to 11, leader changed from d4207c6141b143c590369fc8fef6b316 (127.8.16.66) to 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65). New cstate: current_term: 11 leader_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: UNKNOWN } } }
14:22:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:58.276561 10165 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:22:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:58.319621 10415 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Refusing update from remote peer 84e8522e549f4f23b3fd7f83497bb3ab: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
14:22:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:22:58.320672 10481 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [LEADER]: Connected to new peer: Peer: permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
14:22:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:22:59.785892 10165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:23:00.221 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:00.222 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:00.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3780), Traces: [0ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [2ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [2ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [17ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [19ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [21ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [21ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [37ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [39ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [40ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [41ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [58ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [60ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [62ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [62ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [178ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [179ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [181ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [181ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [315ms] refreshing cache from master, [315ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [318ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [320ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [321ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [321ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [994ms] refreshing cache from master, [994ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [998ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [999ms] sending RPC to server d4207c6141b143c590369fc8fef6b316, [1001ms] received response from server d4207c6141b143c590369fc8fef6b316: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [1001ms] delaying RPC due to: Network error: Failed to connect to peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745): Connection refused: /127.8.16.66:42745, [3775ms] refreshing cache from master, [3775ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [3778ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1715605874(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@4f23d3a1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610) -> (continuation of Deferred@1502706929 after retry RPC@1760246875) -> (continuation of Deferred@24325185 after retry RPC@810275280) -> (continuation of Deferred@734085304 after retry RPC@1173016897), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fef35c1 -> wakeup thread Time-limited test -> (continuation of Deferred@188352909 after retry RPC@1028482070) -> (continuation of Deferred@1790444467 after retry RPC@1006872358) -> (continuation of Deferred@676465401 after retry RPC@155820610) -> (continuation of Deferred@1502706929 after retry RPC@1760246875) -> (continuation of Deferred@24325185 after retry RPC@810275280) -> (continuation of Deferred@734085304 after retry RPC@1173016897))) after lookup
14:23:00.223 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:00.225 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] Successfully connected to peer
14:23:00.248 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xecda9144, L:/127.0.0.1:51154 - R:/127.8.16.65:35141] using TOKEN/null
14:23:00.289 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.66:42745
14:23:00.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:23:00.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.290663  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:23:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:23:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:23:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:23:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:23:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:23:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:23:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:23:00.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:23:00.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:23:00.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:23:00.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:23:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.707340 10519 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:23:00.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.707774 10519 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:23:00.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.707906 10519 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:23:00.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.720871 10519 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:23:00.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.721175 10519 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.66
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.734828 10519 tablet_server_runner.cc:78] Tablet server non-default flags:
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.66:42745
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.66
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35077
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:23:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.66
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/logs
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:23:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:23:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.737021 10519 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:23:00.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.739123 10519 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:23:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.752339 10526 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:23:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.752650 10527 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:23:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.753214 10529 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:23:00.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.755084 10519 server_base.cc:1061] running on GCE node
14:23:00.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.755934 10519 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:23:00.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.757220 10519 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:23:00.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.758502 10519 hybrid_clock.cc:648] HybridClock initialized: now 1779114180758432 us; error 68 us; skew 500 ppm
14:23:00.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.758823 10519 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:23:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.763659 10519 webserver.cc:533] Webserver started at http://127.8.16.66:35077/ using document root <none> and password file <none>
14:23:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.764505 10519 fs_manager.cc:362] Metadata directory not provided
14:23:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.764614 10519 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:23:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.770809 10519 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
14:23:00.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.774163 10535 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:23:00.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.775897 10519 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.776161 10519 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4207c6141b143c590369fc8fef6b316"
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:26 on dist-test-slave-2txq"
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.776986 10519 fs_report.cc:389] FS layout report
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/wal
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/data
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:23:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:23:00.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:23:00.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:23:00.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:23:00.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.807337 10519 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:23:00.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.808470 10519 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:23:00.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.808871 10519 kserver.cc:163] Server-wide thread pool size limit: 3276
14:23:00.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.810174 10519 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:23:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.812777 10542 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:23:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.819772 10519 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:23:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.820004 10519 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
14:23:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.820168 10519 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:23:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.823815 10519 ts_tablet_manager.cc:616] Registered 1 tablets
14:23:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.824118 10519 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
14:23:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.824573 10542 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap starting.
14:23:00.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:00.852677 10165 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:23:00.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.877375 10542 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Log is configured to *not* fsync() on all Append() calls
14:23:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.895876 10519 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.66:42745
14:23:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.896873 10651 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.66:42745 every 8 connection(s)
14:23:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.898478 10519 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-1/data/info.pb
14:23:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.901683  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 10519
14:23:00.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:23:00.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.947950 10542 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:23:00.957 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:23:00.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.958027 10542 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Bootstrap complete.
14:23:00.958 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.8.16.65:35141
14:23:00.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:23:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.959501  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 10150
14:23:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.959642 10542 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent bootstrapping tablet: real 0.136s	user 0.078s	sys 0.042s
14:23:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.968587 10653 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:23:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.969724 10653 heartbeater.cc:461] Registering TS with master...
14:23:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.971477 10652 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:23:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.971607 10652 heartbeater.cc:461] Registering TS with master...
14:23:00.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.973327 10652 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:23:00.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.975443 10653 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:23:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.976998  8300 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:23:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.978888  8441 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:23:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.981395 10542 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.986714 10542 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4207c6141b143c590369fc8fef6b316, State: Initialized, Role: FOLLOWER
14:23:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.987489 10274 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:00.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.987695 10542 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:00.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.989588 10657 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:23:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.989895 10657 heartbeater.cc:461] Registering TS with master...
14:23:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.990828 10657 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:23:00.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.992997  8371 ts_manager.cc:194] Re-registered known tserver with Master: d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745)
14:23:00.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.994935  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.66:51435
14:23:00.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.995023 10542 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316: Time spent starting tablet: real 0.035s	user 0.012s	sys 0.007s
14:23:00.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:00.998097 10657 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:23:01.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:01.098052  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 10150
14:23:01.118 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:23:01.118 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] handling channelInactive
14:23:01.118 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state READY due to: connection closed
14:23:01.119 [DEBUG - Time-limited test] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.120 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.120 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.120 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:01.140 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:01.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test)) after lookup
14:23:01.141 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.142 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.142 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.142 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.159 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:01.160 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:01.160 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [23ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [24ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [24ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [41ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849))) after lookup
14:23:01.161 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.162 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.162 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.162 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:01.180 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:01.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [23ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [24ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [24ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [41ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [42ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [44ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [44ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [61ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212))) after lookup
14:23:01.181 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.182 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.182 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.182 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:01.301 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:01.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=183), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [23ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [24ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [24ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [41ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [42ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [44ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [44ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [61ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [63ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [64ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [64ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [178ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [182ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804))) after lookup
14:23:01.302 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.302 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.303 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.303 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:01.320 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:01.321 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=202), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [23ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [24ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [24ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [41ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [42ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [44ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [44ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [61ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [63ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [64ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [64ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [178ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [182ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [183ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [185ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [185ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [198ms] refreshing cache from master, [199ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [201ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804) -> (continuation of Deferred@17840249 after retry RPC@954561111), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804) -> (continuation of Deferred@17840249 after retry RPC@954561111))) after lookup
14:23:01.321 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.322 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.322 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.323 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.680 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:01.680 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)[L],d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)]}]
14:23:01.681 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=562), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [23ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [24ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [24ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [41ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [42ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [44ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [44ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [61ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [63ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [64ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [64ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [178ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [182ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [183ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [185ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [185ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [198ms] refreshing cache from master, [199ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [201ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [203ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [205ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [205ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [558ms] refreshing cache from master, [558ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [562ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804) -> (continuation of Deferred@17840249 after retry RPC@954561111) -> (continuation of Deferred@194344806 after retry RPC@418503561), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804) -> (continuation of Deferred@17840249 after retry RPC@954561111) -> (continuation of Deferred@194344806 after retry RPC@418503561))) after lookup
14:23:01.682 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] connecting to peer
14:23:01.683 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.683 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:01.683 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141) for tablet 81030604361040b5ac097199a6d3bcdc: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141
14:23:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.280283 10666 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 84e8522e549f4f23b3fd7f83497bb3ab)
14:23:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.280680 10666 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.282565 10666 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:23:02.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:02.286259 10348 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:23:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.294729 10606 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316" is_pre_election: true
14:23:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.295853 10606 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 10.
14:23:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.298079 10347 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab
14:23:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.298478 10666 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Leader pre-election won for term 12
14:23:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.298666 10666 raft_consensus.cc:493] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Starting leader election (detected failure of leader 84e8522e549f4f23b3fd7f83497bb3ab)
14:23:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.298835 10666 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 11 FOLLOWER]: Advancing to term 12
14:23:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.301127 10666 raft_consensus.cc:515] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.302026 10666 leader_election.cc:290] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 12 election: Requested vote from peers d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745), 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:23:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.302870 10606 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "81030604361040b5ac097199a6d3bcdc" candidate_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "d4207c6141b143c590369fc8fef6b316"
14:23:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.303141 10606 raft_consensus.cc:3060] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 10 FOLLOWER]: Advancing to term 12
14:23:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:02.304037 10348 leader_election.cc:336] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111)
14:23:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.307358 10606 raft_consensus.cc:2468] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df3bbabb6b104a5e8e63dca4a47eb6b1 in term 12.
14:23:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.308171 10347 leader_election.cc:304] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d4207c6141b143c590369fc8fef6b316, df3bbabb6b104a5e8e63dca4a47eb6b1; no voters: 84e8522e549f4f23b3fd7f83497bb3ab
14:23:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.308640 10666 raft_consensus.cc:2804] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 12 FOLLOWER]: Leader election won for term 12
14:23:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.309183 10666 raft_consensus.cc:697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [term 12 LEADER]: Becoming Leader. State: Replica: df3bbabb6b104a5e8e63dca4a47eb6b1, State: Running, Role: LEADER
14:23:02.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.309792 10666 consensus_queue.cc:237] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.315150  8371 catalog_manager.cc:5697] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 reported cstate change: term changed from 11 to 12, leader changed from 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65) to df3bbabb6b104a5e8e63dca4a47eb6b1 (127.8.16.67). New cstate: current_term: 12 leader_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } health_report { overall_health: HEALTHY } } }
14:23:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.809494 10606 raft_consensus.cc:1275] T 81030604361040b5ac097199a6d3bcdc P d4207c6141b143c590369fc8fef6b316 [term 12 FOLLOWER]: Refusing update from remote peer df3bbabb6b104a5e8e63dca4a47eb6b1: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
14:23:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:02.810631 10666 consensus_queue.cc:1048] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
14:23:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:02.832567 10348 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111) [suppressed 2 similar messages]
14:23:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:02.835260 10348 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Couldn't send request to peer 84e8522e549f4f23b3fd7f83497bb3ab. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:23:05.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:05.358633 10348 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141): Couldn't send request to peer 84e8522e549f4f23b3fd7f83497bb3ab. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:23:05.540 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81030604361040b5ac097199a6d3bcdc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434' with partition [<start>, <end>)
14:23:05.540 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=81030604361040b5ac097199a6d3bcdc@[84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141),d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745),df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)[L]]}]
14:23:05.541 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=81030604361040b5ac097199a6d3bcdc, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4422), Traces: [0ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [2ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [21ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [23ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [24ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [24ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [41ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [42ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [44ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [44ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [61ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [63ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [64ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [64ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [178ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [182ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [183ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [185ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [185ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [198ms] refreshing cache from master, [199ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [201ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [203ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [205ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [205ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [558ms] refreshing cache from master, [558ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [562ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, [563ms] sending RPC to server 84e8522e549f4f23b3fd7f83497bb3ab, [565ms] received response from server 84e8522e549f4f23b3fd7f83497bb3ab: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [565ms] delaying RPC due to: Network error: Failed to connect to peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141): Connection refused: /127.8.16.65:35141, [4419ms] refreshing cache from master, [4419ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.16.125:36273, [4422ms] Sub RPC GetTableLocations: received response from server master-127.8.16.125:36273: OK, deferred=Deferred@1274210489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f77f036c9f8@5801585f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804) -> (continuation of Deferred@17840249 after retry RPC@954561111) -> (continuation of Deferred@194344806 after retry RPC@418503561) -> (continuation of Deferred@305545057 after retry RPC@67994429), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@579f6619 -> wakeup thread Time-limited test -> (continuation of Deferred@237708155 after retry RPC@573059849) -> (continuation of Deferred@1964549533 after retry RPC@382427212) -> (continuation of Deferred@1240137245 after retry RPC@74333804) -> (continuation of Deferred@17840249 after retry RPC@954561111) -> (continuation of Deferred@194344806 after retry RPC@418503561) -> (continuation of Deferred@305545057 after retry RPC@67994429))) after lookup
14:23:05.542 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] connecting to peer
14:23:05.543 [DEBUG - kudu-nio-3] (Connection.java:757) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] Successfully connected to peer
14:23:05.566 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x2a31f537, L:/127.0.0.1:58040 - R:/127.8.16.67:42073] using TOKEN/null
14:23:05.597 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.8.16.65:35141
14:23:05.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:05.599097  8257 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:23:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:35141
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34929
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:23:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:23:05.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:05.970027 10709 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:23:05.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:05.970432 10709 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:23:05.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:05.970543 10709 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:23:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:05.979642 10709 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:23:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:05.979920 10709 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.16.65
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:05.991235 10709 tablet_server_runner.cc:78] Tablet server non-default flags:
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.16.84:45421
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:23:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.16.65:35141
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.16.65
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34929
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.16.126:46859,127.8.16.125:36273,127.8.16.124:41323
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.16.65
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/logs
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bee9a2d051767e07c70d440ae169cb7177a10f8
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 14:03:53 UTC on e7f111948823
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 15853
14:23:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:23:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:05.993384 10709 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:23:05.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:05.995290 10709 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:23:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:06.006582 10714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:23:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:06.006556 10715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:23:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:06.007017 10717 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:23:06.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.008329 10709 server_base.cc:1061] running on GCE node
14:23:06.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.009136 10709 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:23:06.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.010339 10709 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:23:06.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.011631 10709 hybrid_clock.cc:648] HybridClock initialized: now 1779114186011552 us; error 65 us; skew 500 ppm
14:23:06.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.012080 10709 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:23:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.014792 10709 webserver.cc:533] Webserver started at http://127.8.16.65:34929/ using document root <none> and password file <none>
14:23:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.015754 10709 fs_manager.cc:362] Metadata directory not provided
14:23:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.015893 10709 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:23:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.022658 10709 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
14:23:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.026147 10723 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.027731 10709 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.027897 10709 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data,/tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84e8522e549f4f23b3fd7f83497bb3ab"
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 14:22:25 on dist-test-slave-2txq"
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.028613 10709 fs_report.cc:389] FS layout report
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/wal
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/data
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:23:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:23:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:23:06.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.047250 10709 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:23:06.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.048135 10709 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:23:06.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.048389 10709 kserver.cc:163] Server-wide thread pool size limit: 3276
14:23:06.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.049420 10709 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:23:06.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.051784 10730 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:23:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.058661 10709 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:23:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.058797 10709 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:23:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.058983 10709 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:23:06.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.063009 10709 ts_tablet_manager.cc:616] Registered 1 tablets
14:23:06.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.063139 10709 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
14:23:06.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.063756 10730 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap starting.
14:23:06.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.095903 10730 log.cc:826] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Log is configured to *not* fsync() on all Append() calls
14:23:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.120188 10709 rpc_server.cc:307] RPC server started. Bound to: 127.8.16.65:35141
14:23:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.120270 10838 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.16.65:35141 every 8 connection(s)
14:23:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.121866 10709 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskyeLMhy/test-tmp/mini-kudu-cluster16965391569337307870/ts-0/data/info.pb
14:23:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.132369  8257 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu as pid 10709
14:23:06.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:23:06.147 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:23:06.156 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779114147434, tablet=81030604361040b5ac097199a6d3bcdc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:23:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.165045 10730 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:23:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.166093 10730 tablet_bootstrap.cc:492] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Bootstrap complete.
14:23:06.167 [DEBUG - main] (Connection.java:575) [peer master-127.8.16.126:46859(127.8.16.126:46859)] disconnecting while in state READY
14:23:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.167869 10730 ts_tablet_manager.cc:1403] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent bootstrapping tablet: real 0.105s	user 0.069s	sys 0.016s
14:23:06.171 [DEBUG - main] (Connection.java:575) [peer d4207c6141b143c590369fc8fef6b316(127.8.16.66:42745)] disconnecting while in state TERMINATED
14:23:06.171 [DEBUG - main] (Connection.java:575) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] disconnecting while in state READY
14:23:06.173 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.16.126:46859(127.8.16.126:46859)] handling channelInactive
14:23:06.173 [DEBUG - kudu-nio-3] (Connection.java:239) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] handling channelInactive
14:23:06.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.16.126:46859(127.8.16.126:46859)] cleaning up while in state READY due to: connection closed
14:23:06.173 [DEBUG - main] (Connection.java:575) [peer master-127.8.16.125:36273(127.8.16.125:36273)] disconnecting while in state READY
14:23:06.173 [DEBUG - main] (Connection.java:575) [peer 84e8522e549f4f23b3fd7f83497bb3ab(127.8.16.65:35141)] disconnecting while in state TERMINATED
14:23:06.174 [DEBUG - main] (Connection.java:575) [peer master-127.8.16.124:41323(127.8.16.124:41323)] disconnecting while in state READY
14:23:06.174 [DEBUG - kudu-nio-3] (Connection.java:709) [peer df3bbabb6b104a5e8e63dca4a47eb6b1(127.8.16.67:42073)] cleaning up while in state READY due to: connection closed
14:23:06.174 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.8.16.124:41323(127.8.16.124:41323)] handling channelInactive
14:23:06.174 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.8.16.124:41323(127.8.16.124:41323)] cleaning up while in state READY due to: connection closed
14:23:06.174 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.16.125:36273(127.8.16.125:36273)] handling channelInactive
14:23:06.175 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.16.125:36273(127.8.16.125:36273)] cleaning up while in state READY due to: connection closed
14:23:06.174 [DEBUG - kudu-nio-3] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:23:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.182860  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 10709
14:23:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.183588 10730 raft_consensus.cc:359] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.184638 10730 raft_consensus.cc:740] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84e8522e549f4f23b3fd7f83497bb3ab, State: Initialized, Role: FOLLOWER
14:23:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.185487 10730 consensus_queue.cc:260] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d4207c6141b143c590369fc8fef6b316" member_type: VOTER last_known_addr { host: "127.8.16.66" port: 42745 } } peers { permanent_uuid: "84e8522e549f4f23b3fd7f83497bb3ab" member_type: VOTER last_known_addr { host: "127.8.16.65" port: 35141 } } peers { permanent_uuid: "df3bbabb6b104a5e8e63dca4a47eb6b1" member_type: VOTER last_known_addr { host: "127.8.16.67" port: 42073 } }
14:23:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.191995 10730 ts_tablet_manager.cc:1434] T 81030604361040b5ac097199a6d3bcdc P 84e8522e549f4f23b3fd7f83497bb3ab: Time spent starting tablet: real 0.024s	user 0.005s	sys 0.007s
14:23:06.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.205991 10841 heartbeater.cc:344] Connected to a master server at 127.8.16.125:36273
14:23:06.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.206542 10841 heartbeater.cc:461] Registering TS with master...
14:23:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.206583 10839 heartbeater.cc:344] Connected to a master server at 127.8.16.126:46859
14:23:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.206743 10839 heartbeater.cc:461] Registering TS with master...
14:23:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.207274 10840 heartbeater.cc:344] Connected to a master server at 127.8.16.124:41323
14:23:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.207412 10840 heartbeater.cc:461] Registering TS with master...
14:23:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.208451 10841 heartbeater.cc:507] Master 127.8.16.125:36273 requested a full tablet report, sending...
14:23:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.208642 10839 heartbeater.cc:507] Master 127.8.16.126:46859 requested a full tablet report, sending...
14:23:06.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.210459 10840 heartbeater.cc:507] Master 127.8.16.124:41323 requested a full tablet report, sending...
14:23:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.212419  8442 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:23:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.213116  8299 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:23:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.213455  8371 ts_manager.cc:194] Re-registered known tserver with Master: 84e8522e549f4f23b3fd7f83497bb3ab (127.8.16.65:35141)
14:23:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.215031  8371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.16.65:49507
14:23:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.217935 10834 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:06.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.218976 10841 heartbeater.cc:499] Master 127.8.16.125:36273 was elected leader, sending a full tablet report...
14:23:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.345330  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 10709
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.364554  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 10519
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:06.365360 10697 negotiation.cc:336] Failed RPC negotiation. Trace:
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.250449 (+     0us) reactor.cc:678] Submitting negotiation task for client connection to 127.8.16.65:35141 (local address 127.8.16.67:42445)
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.250615 (+   166us) negotiation.cc:107] Waiting for socket to connect
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.250641 (+    26us) client_negotiation.cc:175] Beginning negotiation
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.250753 (+   112us) client_negotiation.cc:253] Sending NEGOTIATE NegotiatePB request
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.343581 (+ 92828us) client_negotiation.cc:272] Received NEGOTIATE NegotiatePB response
14:23:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.343591 (+    10us) client_negotiation.cc:395] Received NEGOTIATE response from server
14:23:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.343632 (+    41us) client_negotiation.cc:190] Negotiated authn=CERTIFICATE
14:23:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.344381 (+   749us) client_negotiation.cc:515] Sending TLS_HANDSHAKE message to server
14:23:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.344398 (+    17us) client_negotiation.cc:253] Sending TLS_HANDSHAKE NegotiatePB request
14:23:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 14:23:06.364136 (+ 19738us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.8.16.65:35141: BlockingRecv error: recv got EOF from 127.8.16.65:35141 (error 108)
14:23:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":50}
14:23:06.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.373932 10647 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.472044  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 10519
14:23:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:06.492847 10347 connection.cc:582] client connection to 127.8.16.66:42745 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:23:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.493002  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 10332
14:23:06.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 14:23:06.495697 10347 consensus_peers.cc:597] T 81030604361040b5ac097199a6d3bcdc P df3bbabb6b104a5e8e63dca4a47eb6b1 -> Peer d4207c6141b143c590369fc8fef6b316 (127.8.16.66:42745): Couldn't send request to peer d4207c6141b143c590369fc8fef6b316. Status: Network error: Client connection negotiation failed: client connection to 127.8.16.66:42745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:23:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.503121 10456 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:06.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.640167  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 10332
14:23:06.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.662575  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 8269
14:23:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.663810  8330 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.785490  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 8269
14:23:06.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.809082  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 8337
14:23:06.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.810539  8401 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.934427  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 8337
14:23:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.955062  8257 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu pid 8408
14:23:06.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:06.956477  8472 generic_service.cc:196] Checking for leaks (request via RPC)
14:23:07.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 14:23:07.077342  8257 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskyeLMhy/build/asan/bin/kudu with pid 8408
14:23:07.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T14:23:07Z chronyd exiting

Time: 45.524

OK (1 test)

