JUnit version 4.13.2
.23:51:47.806 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783468307799
23:51:47.861 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
23:51:47.892 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:51:47.902 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskUpAgIg/test-tmp
23:51:47.928 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
23:51:47.931 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskUpAgIg/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
23:51:48.129 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

23:51:48.145 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:51:48.146 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

23:51:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.148063 23157 mini_oidc.cc:150] Starting JWKS server
23:51:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.150219 23157 webserver.cc:534] Webserver started at http://127.0.0.1:45969/ using document root <none> and password file <none>
23:51:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.150409 23157 mini_oidc.cc:186] Starting OIDC Discovery server
23:51:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.150676 23157 webserver.cc:534] Webserver started at http://127.0.0.1:35191/ using document root <none> and password file <none>
23:51:48.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:51:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
23:51:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:51:48Z Disabled control of system clock
23:51:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.169112 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal
23:51:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data
23:51:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:48.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:48.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/logs
23:51:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data/info.pb
23:51:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:48.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:48.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:48.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.126:38363
23:51:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.126
23:51:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45969/jwks.json with env {}
23:51:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.272800 23168 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.273046 23168 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:48.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.273108 23168 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:48.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.276816 23168 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:51:48.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.276904 23168 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:48.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.276935 23168 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:48.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.276962 23168 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:51:48.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.276994 23168 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:48.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.277019 23168 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45969/jwks.json
23:51:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.281260 23168 master_runner.cc:387] Master server non-default flags:
23:51:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data
23:51:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal
23:51:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.126:38363
23:51:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45969/jwks.json
23:51:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data/info.pb
23:51:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.126
23:51:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23168
23:51:48.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:48.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:48.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/logs
23:51:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:48.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.282716 23168 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:48.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.284212 23168 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.290933 23176 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.290911 23173 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.290918 23174 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.291461 23168 server_base.cc:1061] running on GCE node
23:51:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.292949 23168 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.293872 23168 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.295053 23168 hybrid_clock.cc:648] HybridClock initialized: now 1783468308295038 us; error 27 us; skew 500 ppm
23:51:48.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.296825 23168 webserver.cc:534] Webserver started at http://127.22.157.126:33139/ using document root <none> and password file <none>
23:51:48.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.297456 23168 fs_manager.cc:362] Metadata directory not provided
23:51:48.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.297544 23168 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:48.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.297780 23168 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:48.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.299357 23168 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data/instance:
23:51:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6671afb7dcd045fb892a7989dacb21cf"
23:51:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.299944 23168 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal/instance:
23:51:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6671afb7dcd045fb892a7989dacb21cf"
23:51:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.303306 23168 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
23:51:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.305472 23182 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.306506 23168 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:51:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.306608 23168 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal
23:51:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6671afb7dcd045fb892a7989dacb21cf"
23:51:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.306674 23168 fs_report.cc:389] FS layout report
23:51:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal
23:51:48.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal
23:51:48.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data/data
23:51:48.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:48.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.325131 23168 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.325820 23168 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.326009 23168 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:48.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.333174 23168 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.126:38363
23:51:48.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.333189 23234 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.126:38363 every 8 connection(s)
23:51:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.334108 23168 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/data/info.pb
23:51:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.336442 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23168
23:51:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.336588 23157 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-0/wal/instance
23:51:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.337275 23235 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.338205 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal
23:51:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/logs
23:51:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data/info.pb
23:51:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.125:42957
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.125
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45969/jwks.json with env {}
23:51:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.340780 23235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.352862 23235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.353747 23186 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.125:42957: connect: Connection refused (error 111)
23:51:48.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.354764 23235 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.125:42957: Network error: Client connection negotiation failed: client connection to 127.22.157.125:42957: connect: Connection refused (error 111)
23:51:48.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.404007 23235 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } attempt: 1
23:51:48.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.405854 23235 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.125:42957: Network error: Client connection negotiation failed: client connection to 127.22.157.125:42957: connect: Connection refused (error 111)
23:51:48.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.444891 23237 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.445111 23237 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.445156 23237 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.448865 23237 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:51:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.448932 23237 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.448953 23237 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.448971 23237 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:51:48.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.448989 23237 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:48.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.449004 23237 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45969/jwks.json
23:51:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.454002 23237 master_runner.cc:387] Master server non-default flags:
23:51:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.125:42957
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45969/jwks.json
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data/info.pb
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.125
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23237
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/logs
23:51:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.455188 23237 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.456631 23237 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:48.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.462599 23245 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.462599 23248 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.462941 23246 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.463230 23237 server_base.cc:1061] running on GCE node
23:51:48.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.463824 23237 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.464973 23237 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:48.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.466580 23237 hybrid_clock.cc:648] HybridClock initialized: now 1783468308466519 us; error 78 us; skew 500 ppm
23:51:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.468469 23237 webserver.cc:534] Webserver started at http://127.22.157.125:38951/ using document root <none> and password file <none>
23:51:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.469069 23237 fs_manager.cc:362] Metadata directory not provided
23:51:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.469125 23237 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.469282 23237 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.471108 23237 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data/instance:
23:51:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61983681e12044ea91888e7fd89b23ab"
23:51:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.471650 23237 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal/instance:
23:51:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61983681e12044ea91888e7fd89b23ab"
23:51:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.474061 23235 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } attempt: 2
23:51:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.475629 23237 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.002s	sys 0.001s
23:51:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.475631 23235 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.125:42957: Network error: Client connection negotiation failed: client connection to 127.22.157.125:42957: connect: Connection refused (error 111)
23:51:48.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.478403 23254 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:48.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.479490 23237 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
23:51:48.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.479594 23237 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal
23:51:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61983681e12044ea91888e7fd89b23ab"
23:51:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.479682 23237 fs_report.cc:389] FS layout report
23:51:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data/data
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.507889 23237 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.508797 23237 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.509037 23237 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.516764 23237 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.125:42957
23:51:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.516794 23308 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.125:42957 every 8 connection(s)
23:51:48.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.517930 23237 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/data/info.pb
23:51:48.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.520839 23309 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:48.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.523552 23309 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.526481 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23237
23:51:48.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.526582 23157 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-1/wal/instance
23:51:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.527266 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal
23:51:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data
23:51:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/logs
23:51:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data/info.pb
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.124:43683
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.124
23:51:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45969/jwks.json with env {}
23:51:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.538064 23309 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.545352 23309 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.546211 23258 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.547356 23309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.124:43683: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.566810 23235 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } attempt: 3
23:51:48.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.571516 23235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.572942 23235 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.124:43683: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.596591 23309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } attempt: 1
23:51:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.598145 23309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.124:43683: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.604106 23235 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } attempt: 1
23:51:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.605552 23235 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.124:43683: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.635609 23312 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.635829 23312 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.635876 23312 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.639163 23312 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:51:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.639218 23312 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.639238 23312 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.639256 23312 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:51:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.639273 23312 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.639287 23312 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45969/jwks.json
23:51:48.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.643302 23312 master_runner.cc:387] Master server non-default flags:
23:51:48.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.124:43683
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45969/jwks.json
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data/info.pb
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.124
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23312
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/logs
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:48.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:48.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.644501 23312 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:48.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.645641 23312 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:48.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.651980 23319 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.651980 23320 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.651978 23322 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.653121 23312 server_base.cc:1061] running on GCE node
23:51:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.653738 23312 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.654768 23312 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.655939 23312 hybrid_clock.cc:648] HybridClock initialized: now 1783468308655886 us; error 72 us; skew 500 ppm
23:51:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.657650 23312 webserver.cc:534] Webserver started at http://127.22.157.124:40457/ using document root <none> and password file <none>
23:51:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.658258 23312 fs_manager.cc:362] Metadata directory not provided
23:51:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.658339 23312 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.658581 23312 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.660252 23312 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data/instance:
23:51:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "050829536e0445b7b5b75468c7b2e798"
23:51:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.660795 23312 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal/instance:
23:51:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "050829536e0445b7b5b75468c7b2e798"
23:51:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.664220 23312 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.001s
23:51:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.666275 23328 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.666314 23309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } attempt: 2
23:51:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.667505 23312 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
23:51:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.667625 23312 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "050829536e0445b7b5b75468c7b2e798"
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:48 on dist-test-slave-3q43"
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.667732 23312 fs_report.cc:389] FS layout report
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data/data
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.667843 23309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.124:43683: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.680730 23235 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } attempt: 2
23:51:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.682161 23235 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.157.124:43683: Network error: Client connection negotiation failed: client connection to 127.22.157.124:43683: connect: Connection refused (error 111)
23:51:48.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.689146 23312 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:48.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.689766 23312 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.689909 23312 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.696832 23381 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.124:43683 every 8 connection(s)
23:51:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.696830 23312 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.124:43683
23:51:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.698005 23312 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/data/info.pb
23:51:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.700806 23382 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:48.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.703534 23382 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.704602 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23312
23:51:48.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.704725 23157 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/master-2/wal/instance
23:51:48.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.712782 23382 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.717876 23382 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } has no permanent_uuid. Determining permanent_uuid...
23:51:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.724821 23382 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: Bootstrap starting.
23:51:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.726817 23382 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: Neither blocks nor log segments found. Creating new log.
23:51:48.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.727486 23382 log.cc:826] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: Log is configured to *not* fsync() on all Append() calls
23:51:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.729149 23382 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: No bootstrap required, opened a new log
23:51:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.731992 23382 raft_consensus.cc:359] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.732453 23382 raft_consensus.cc:385] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.732514 23382 raft_consensus.cc:740] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 050829536e0445b7b5b75468c7b2e798, State: Initialized, Role: FOLLOWER
23:51:48.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.732841 23382 consensus_queue.cc:260] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:48.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.733363 23391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:48.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.733486 23391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: This master's current role is: FOLLOWER
23:51:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.733982 23382 sys_catalog.cc:565] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: configured and running, proceeding with master startup.
23:51:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.738885 23402 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.739207 23402 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.737252 23312 master_runner.cc:187] Error getting master registration for 127.22.157.126:38363: OK, instance_id { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" instance_seqno: 1783468308331279 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.739432 23312 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:51:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.759006 23309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } attempt: 3
23:51:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.765256 23309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: Bootstrap starting.
23:51:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.767555 23309 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: Neither blocks nor log segments found. Creating new log.
23:51:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.768592 23309 log.cc:826] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: Log is configured to *not* fsync() on all Append() calls
23:51:48.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.770462 23309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: No bootstrap required, opened a new log
23:51:48.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.773681 23309 raft_consensus.cc:359] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.773855 23309 raft_consensus.cc:385] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.773903 23309 raft_consensus.cc:740] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61983681e12044ea91888e7fd89b23ab, State: Initialized, Role: FOLLOWER
23:51:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.774325 23309 consensus_queue.cc:260] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.774750 23405 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.774879 23405 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: This master's current role is: FOLLOWER
23:51:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.775498 23309 sys_catalog.cc:565] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: configured and running, proceeding with master startup.
23:51:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.780668 23416 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.781037 23416 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.781307 23235 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } attempt: 3
23:51:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.779060 23237 master_runner.cc:187] Error getting master registration for 127.22.157.126:38363: OK, instance_id { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" instance_seqno: 1783468308331279 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.783499 23237 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:51:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.788559 23235 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: Bootstrap starting.
23:51:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.791203 23235 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: Neither blocks nor log segments found. Creating new log.
23:51:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.792047 23235 log.cc:826] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: Log is configured to *not* fsync() on all Append() calls
23:51:48.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.793974 23235 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: No bootstrap required, opened a new log
23:51:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.797549 23235 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.797972 23235 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.798043 23235 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6671afb7dcd045fb892a7989dacb21cf, State: Initialized, Role: FOLLOWER
23:51:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.798471 23235 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.799082 23419 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.799190 23419 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: This master's current role is: FOLLOWER
23:51:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.799677 23235 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: configured and running, proceeding with master startup.
23:51:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.804461 23430 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.804781 23430 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:48.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.812911 23168 master_runner.cc:264] No leader master found from master 6671afb7dcd045fb892a7989dacb21cf
23:51:48.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:48.812988 23168 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:51:48.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.827435 23188 tablet.cc:2406] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:48.941064 23334 tablet.cc:2406] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.010800 23260 tablet.cc:2406] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:49.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.023782 23419 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:49.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.023991 23419 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:49.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.025230 23419 leader_election.cc:290] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 61983681e12044ea91888e7fd89b23ab (127.22.157.125:42957), 050829536e0445b7b5b75468c7b2e798 (127.22.157.124:43683)
23:51:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.025978 23282 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6671afb7dcd045fb892a7989dacb21cf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61983681e12044ea91888e7fd89b23ab" is_pre_election: true
23:51:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.025977 23355 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6671afb7dcd045fb892a7989dacb21cf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "050829536e0445b7b5b75468c7b2e798" is_pre_election: true
23:51:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.026255 23355 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6671afb7dcd045fb892a7989dacb21cf in term 0.
23:51:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.026255 23282 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6671afb7dcd045fb892a7989dacb21cf in term 0.
23:51:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.026683 23186 leader_election.cc:304] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 61983681e12044ea91888e7fd89b23ab, 6671afb7dcd045fb892a7989dacb21cf; no voters: 
23:51:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.026902 23419 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Leader pre-election won for term 1
23:51:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.026962 23419 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:51:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.027002 23419 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 0 FOLLOWER]: Advancing to term 1
23:51:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.028121 23419 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.028505 23419 leader_election.cc:290] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [CANDIDATE]: Term 1 election: Requested vote from peers 61983681e12044ea91888e7fd89b23ab (127.22.157.125:42957), 050829536e0445b7b5b75468c7b2e798 (127.22.157.124:43683)
23:51:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.028882 23282 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6671afb7dcd045fb892a7989dacb21cf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61983681e12044ea91888e7fd89b23ab"
23:51:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.028999 23282 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 0 FOLLOWER]: Advancing to term 1
23:51:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.028980 23355 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6671afb7dcd045fb892a7989dacb21cf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "050829536e0445b7b5b75468c7b2e798"
23:51:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.029129 23355 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 0 FOLLOWER]: Advancing to term 1
23:51:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.029855 23282 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6671afb7dcd045fb892a7989dacb21cf in term 1.
23:51:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.030257 23355 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6671afb7dcd045fb892a7989dacb21cf in term 1.
23:51:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.030257 23186 leader_election.cc:304] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 61983681e12044ea91888e7fd89b23ab, 6671afb7dcd045fb892a7989dacb21cf; no voters: 
23:51:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.030475 23419 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 1 FOLLOWER]: Leader election won for term 1
23:51:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.030638 23419 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [term 1 LEADER]: Becoming Leader. State: Replica: 6671afb7dcd045fb892a7989dacb21cf, State: Running, Role: LEADER
23:51:49.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.030932 23419 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } }
23:51:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.032972 23432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6671afb7dcd045fb892a7989dacb21cf. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.033774 23432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: This master's current role is: LEADER
23:51:49.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.034168 23434 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
23:51:49.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.035101 23434 catalog_manager.cc:1519] Initializing Kudu cluster ID...
23:51:49.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.040541 23355 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [term 1 FOLLOWER]: Refusing update from remote peer 6671afb7dcd045fb892a7989dacb21cf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.040704 23282 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [term 1 FOLLOWER]: Refusing update from remote peer 6671afb7dcd045fb892a7989dacb21cf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.041175 23419 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [LEADER]: Connected to new peer: Peer: permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.041507 23432 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [LEADER]: Connected to new peer: Peer: permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.045223 23405 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6671afb7dcd045fb892a7989dacb21cf. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.045240 23391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6671afb7dcd045fb892a7989dacb21cf. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.045353 23391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: This master's current role is: FOLLOWER
23:51:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.045353 23405 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: This master's current role is: FOLLOWER
23:51:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048048 23432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048174 23432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: This master's current role is: LEADER
23:51:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048333 23432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048444 23432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf [sys.catalog]: This master's current role is: LEADER
23:51:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048440 23405 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048534 23405 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab [sys.catalog]: This master's current role is: FOLLOWER
23:51:49.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048748 23391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6671afb7dcd045fb892a7989dacb21cf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6671afb7dcd045fb892a7989dacb21cf" member_type: VOTER last_known_addr { host: "127.22.157.126" port: 38363 } } peers { permanent_uuid: "61983681e12044ea91888e7fd89b23ab" member_type: VOTER last_known_addr { host: "127.22.157.125" port: 42957 } } peers { permanent_uuid: "050829536e0445b7b5b75468c7b2e798" member_type: VOTER last_known_addr { host: "127.22.157.124" port: 43683 } } }
23:51:49.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.048846 23391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798 [sys.catalog]: This master's current role is: FOLLOWER
23:51:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.051087 23434 catalog_manager.cc:1382] Generated new cluster ID: c543f1c16bda4e1dbfdd17275547da2a
23:51:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.051157 23434 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
23:51:49.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.067565 23434 catalog_manager.cc:1405] Generated new certificate authority record
23:51:49.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.068792 23434 catalog_manager.cc:1539] Loading token signing keys...
23:51:49.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.078433 23434 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 6671afb7dcd045fb892a7989dacb21cf: Generated new TSK 0
23:51:49.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.079200 23434 catalog_manager.cc:1549] Initializing in-progress tserver states...
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.091866 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:0
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.194599 23445 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.194811 23445 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.194854 23445 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.198161 23445 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.198266 23445 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.65
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.202159 23445 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:0
23:51:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23445
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.203298 23445 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.204810 23445 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.212558 23451 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.212580 23450 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.212720 23453 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.213049 23445 server_base.cc:1061] running on GCE node
23:51:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.213358 23445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:49.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.213865 23445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.215034 23445 hybrid_clock.cc:648] HybridClock initialized: now 1783468309215012 us; error 36 us; skew 500 ppm
23:51:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.216930 23445 webserver.cc:534] Webserver started at http://127.22.157.65:40325/ using document root <none> and password file <none>
23:51:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.217540 23445 fs_manager.cc:362] Metadata directory not provided
23:51:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.217623 23445 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.217871 23445 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.219496 23445 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/instance:
23:51:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:51:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.220018 23445 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal/instance:
23:51:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:51:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.223558 23445 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.004s	sys 0.000s
23:51:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.225742 23459 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.226752 23445 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.226869 23445 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.226938 23445 fs_report.cc:389] FS layout report
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/data
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:49.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.247115 23445 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.247803 23445 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.247939 23445 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.248436 23445 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.249362 23445 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.249408 23445 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:51:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.249445 23445 ts_tablet_manager.cc:616] Registered 0 tablets
23:51:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.249460 23445 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:51:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.258687 23445 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.65:39389
23:51:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.258788 23572 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.65:39389 every 8 connection(s)
23:51:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.260274 23445 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:51:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.267503 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23445
23:51:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.267597 23157 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal/instance
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271273 23574 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271319 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:0
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271330 23573 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271562 23575 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271549 23573 heartbeater.cc:461] Registering TS with master...
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271636 23575 heartbeater.cc:461] Registering TS with master...
23:51:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.272220 23575 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:51:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.272226 23573 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:51:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.271549 23574 heartbeater.cc:461] Registering TS with master...
23:51:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.273053 23574 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:51:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.273849 23345 ts_manager.cc:194] Registered new tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:51:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.273845 23272 ts_manager.cc:194] Registered new tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:51:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.274116 23199 ts_manager.cc:194] Registered new tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:51:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.275666 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.65:57211
23:51:49.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.376420 23583 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:49.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.376752 23583 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:49.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.376827 23583 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.380378 23583 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.380548 23583 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.66
23:51:49.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.384440 23583 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:49.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:49.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:0
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23583
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.385874 23583 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:49.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.387327 23583 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.393951 23591 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.394026 23589 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.393971 23588 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.394284 23583 server_base.cc:1061] running on GCE node
23:51:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.394637 23583 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.395145 23583 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.396349 23583 hybrid_clock.cc:648] HybridClock initialized: now 1783468309396309 us; error 47 us; skew 500 ppm
23:51:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.398227 23583 webserver.cc:534] Webserver started at http://127.22.157.66:43755/ using document root <none> and password file <none>
23:51:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.398847 23583 fs_manager.cc:362] Metadata directory not provided
23:51:49.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.398942 23583 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:49.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.399214 23583 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.400943 23583 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/instance:
23:51:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:51:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.401510 23583 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal/instance:
23:51:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:51:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.405040 23583 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.005s	sys 0.000s
23:51:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.407303 23597 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.408428 23583 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:51:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.408520 23583 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.408589 23583 fs_report.cc:389] FS layout report
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/data
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:49.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.424484 23583 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.425082 23583 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.425227 23583 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.425712 23583 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.426661 23583 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.426708 23583 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:51:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.426748 23583 ts_tablet_manager.cc:616] Registered 0 tablets
23:51:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.426761 23583 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:51:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.436285 23583 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.66:37793
23:51:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.436352 23710 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.66:37793 every 8 connection(s)
23:51:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.437402 23583 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:51:49.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.440217 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23583
23:51:49.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.440332 23157 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal/instance
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.442996 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:0
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:51:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:51:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.447654 23712 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:51:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.447906 23712 heartbeater.cc:461] Registering TS with master...
23:51:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.448482 23712 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:51:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.449786 23199 ts_manager.cc:194] Registered new tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:49.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.450394 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.66:50313
23:51:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.451417 23711 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:51:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.451517 23711 heartbeater.cc:461] Registering TS with master...
23:51:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.451678 23713 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:51:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.451757 23713 heartbeater.cc:461] Registering TS with master...
23:51:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.451900 23713 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:51:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.452611 23272 ts_manager.cc:194] Registered new tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.452634 23711 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:51:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.453440 23345 ts_manager.cc:194] Registered new tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:49.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.549340 23718 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:49.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.549597 23718 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:49.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.549664 23718 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:49.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.553252 23718 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:49.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.553404 23718 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.67
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.557415 23718 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:0
23:51:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23718
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.558442 23718 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.559495 23718 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.565950 23723 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.565956 23725 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.566112 23718 server_base.cc:1061] running on GCE node
23:51:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.565950 23727 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.566545 23718 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:49.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.567030 23718 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.568176 23718 hybrid_clock.cc:648] HybridClock initialized: now 1783468309568161 us; error 34 us; skew 500 ppm
23:51:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.569802 23718 webserver.cc:534] Webserver started at http://127.22.157.67:45069/ using document root <none> and password file <none>
23:51:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.570287 23718 fs_manager.cc:362] Metadata directory not provided
23:51:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.570344 23718 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.570554 23718 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.572073 23718 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/instance:
23:51:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:51:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.572526 23718 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal/instance:
23:51:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:51:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.576735 23718 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.004s	sys 0.000s
23:51:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.579826 23733 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.580932 23718 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.581036 23718 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.581108 23718 fs_report.cc:389] FS layout report
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/data
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.597018 23718 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.597615 23718 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.597754 23718 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:49.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.598232 23718 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.599272 23718 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.599326 23718 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:51:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.599428 23718 ts_tablet_manager.cc:616] Registered 0 tablets
23:51:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.599452 23718 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:51:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.609161 23718 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.67:35111
23:51:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.609187 23846 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.67:35111 every 8 connection(s)
23:51:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.610095 23718 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:51:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.611164 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23718
23:51:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.611281 23157 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal/instance
23:51:49.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.619899 23848 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.619899 23847 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.619982 23849 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.620152 23848 heartbeater.cc:461] Registering TS with master...
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.620163 23847 heartbeater.cc:461] Registering TS with master...
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.620164 23849 heartbeater.cc:461] Registering TS with master...
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.620640 23847 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.620640 23848 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.620640 23849 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:51:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.621717 23199 ts_manager.cc:194] Registered new tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.621740 23272 ts_manager.cc:194] Registered new tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.621801 23345 ts_manager.cc:194] Registered new tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.622370 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.67:53373
23:51:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.632184 23157 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
23:51:49.633 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:51:49.634 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

23:51:49.642 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.22.157.126"
      port: 38363
    }
    bound_http_address {
      host: "127.22.157.126"
      port: 33139
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.22.157.125"
      port: 42957
    }
    bound_http_address {
      host: "127.22.157.125"
      port: 38951
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.22.157.124"
      port: 43683
    }
    bound_http_address {
      host: "127.22.157.124"
      port: 40457
    }
  }
}

23:51:49.645 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

23:51:49.648 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.22.157.65"
      port: 39389
    }
    bound_http_address {
      host: "127.22.157.65"
      port: 40325
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.22.157.66"
      port: 37793
    }
    bound_http_address {
      host: "127.22.157.66"
      port: 43755
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.22.157.67"
      port: 35111
    }
    bound_http_address {
      host: "127.22.157.67"
      port: 45069
    }
  }
}

23:51:49.648 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:51:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.741572 23402 catalog_manager.cc:1294] Loaded cluster ID: c543f1c16bda4e1dbfdd17275547da2a
23:51:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.741693 23402 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: loading cluster ID for follower catalog manager: success
23:51:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.743600 23402 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: acquiring CA information for follower catalog manager: success
23:51:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.744812 23402 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 050829536e0445b7b5b75468c7b2e798: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:51:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.782768 23416 catalog_manager.cc:1294] Loaded cluster ID: c543f1c16bda4e1dbfdd17275547da2a
23:51:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.782871 23416 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: loading cluster ID for follower catalog manager: success
23:51:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.785461 23416 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: acquiring CA information for follower catalog manager: success
23:51:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.786913 23416 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 61983681e12044ea91888e7fd89b23ab: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:51:49.828 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
23:51:49.885 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
23:51:50.029 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.157.126:38363(127.22.157.126:38363)] connecting to peer
23:51:50.078 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.157.125:42957(127.22.157.125:42957)] connecting to peer
23:51:50.081 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.157.124:43683(127.22.157.124:43683)] connecting to peer
23:51:50.097 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.157.126:38363(127.22.157.126:38363)] Successfully connected to peer
23:51:50.097 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.22.157.124:43683(127.22.157.124:43683)] Successfully connected to peer
23:51:50.097 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.157.125:42957(127.22.157.125:42957)] Successfully connected to peer
23:51:50.205 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.157.124
23:51:50.205 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.157.126
23:51:50.208 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.157.125
23:51:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.277570 23574 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:51:50.430 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1875af87, L:/127.0.0.1:49466 - R:/127.22.157.126:38363] using SASL/PLAIN
23:51:50.430 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2fc1e014, L:/127.0.0.1:38786 - R:/127.22.157.125:42957] using SASL/PLAIN
23:51:50.430 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x6cf27f58, L:/127.0.0.1:38788 - R:/127.22.157.124:43683] using SASL/PLAIN
23:51:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.452767 23712 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:51:50.466 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
23:51:50.467 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=473), Traces: [0ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.157.126:38363, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.157.125:42957, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.157.124:43683, [445ms] Sub RPC ConnectToMaster: received response from server master-127.22.157.125:42957: OK, [445ms] Sub RPC ConnectToMaster: received response from server master-127.22.157.124:43683: OK, [446ms] Sub RPC ConnectToMaster: received response from server master-127.22.157.126:38363: OK, deferred=null) after lookup
23:51:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.514674 23199 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49466:
23:51:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982"
23:51:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
23:51:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
23:51:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
23:51:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
23:51:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
23:51:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
23:51:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
23:51:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:51:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.517446 23199 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
23:51:50.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 17075ca59c4040bcba28f9134ccae5d7
23:51:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.559911 23781 tablet_service.cc:1511] Processing CreateTablet for tablet 7ece0c8f846e439aaa0a323ed2d0c121 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982 [id=17075ca59c4040bcba28f9134ccae5d7]), partition=RANGE (key) PARTITION UNBOUNDED
23:51:50.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.560446 23507 tablet_service.cc:1511] Processing CreateTablet for tablet 7ece0c8f846e439aaa0a323ed2d0c121 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982 [id=17075ca59c4040bcba28f9134ccae5d7]), partition=RANGE (key) PARTITION UNBOUNDED
23:51:50.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.560454 23645 tablet_service.cc:1511] Processing CreateTablet for tablet 7ece0c8f846e439aaa0a323ed2d0c121 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982 [id=17075ca59c4040bcba28f9134ccae5d7]), partition=RANGE (key) PARTITION UNBOUNDED
23:51:50.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.561348 23781 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ece0c8f846e439aaa0a323ed2d0c121. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:50.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.561388 23507 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ece0c8f846e439aaa0a323ed2d0c121. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:50.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.561388 23645 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ece0c8f846e439aaa0a323ed2d0c121. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:50.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.568897 23878 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap starting.
23:51:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.570606 23877 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap starting.
23:51:50.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.571981 23878 tablet_bootstrap.cc:654] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Neither blocks nor log segments found. Creating new log.
23:51:50.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.572698 23879 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap starting.
23:51:50.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.573014 23877 tablet_bootstrap.cc:654] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Neither blocks nor log segments found. Creating new log.
23:51:50.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.573083 23878 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Log is configured to *not* fsync() on all Append() calls
23:51:50.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.574151 23877 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Log is configured to *not* fsync() on all Append() calls
23:51:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.574918 23879 tablet_bootstrap.cc:654] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Neither blocks nor log segments found. Creating new log.
23:51:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.574947 23878 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: No bootstrap required, opened a new log
23:51:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.575150 23878 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent bootstrapping tablet: real 0.006s	user 0.004s	sys 0.000s
23:51:50.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.575949 23877 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: No bootstrap required, opened a new log
23:51:50.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.576303 23879 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Log is configured to *not* fsync() on all Append() calls
23:51:50.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.576375 23877 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent bootstrapping tablet: real 0.006s	user 0.005s	sys 0.000s
23:51:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.578238 23879 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: No bootstrap required, opened a new log
23:51:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.578415 23879 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent bootstrapping tablet: real 0.006s	user 0.005s	sys 0.000s
23:51:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.578862 23878 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.579088 23878 raft_consensus.cc:385] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.579159 23878 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Initialized, Role: FOLLOWER
23:51:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.579690 23878 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.580685 23878 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent starting tablet: real 0.005s	user 0.000s	sys 0.006s
23:51:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.581144 23877 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.581353 23877 raft_consensus.cc:385] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.581425 23877 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Initialized, Role: FOLLOWER
23:51:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.581845 23877 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.582896 23877 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
23:51:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.586336 23879 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.586575 23879 raft_consensus.cc:385] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.586628 23879 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Initialized, Role: FOLLOWER
23:51:50.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.587649 23879 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.588305 23848 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:51:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.588534 23879 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent starting tablet: real 0.010s	user 0.000s	sys 0.004s
23:51:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.614538 23850 tablet.cc:2406] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.684348 23883 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.684522 23883 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.685880 23883 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.689846 23801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc" is_pre_election: true
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.689934 23665 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613" is_pre_election: true
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.689953 23714 tablet.cc:2406] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.690160 23801 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 0.
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.690207 23665 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 0.
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.690634 23460 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 
23:51:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.690860 23883 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:51:50.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.690925 23883 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:51:50.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.690981 23883 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 0 FOLLOWER]: Advancing to term 1
23:51:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.691982 23883 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.692353 23883 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 1 election: Requested vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.692667 23665 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:51:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.692757 23801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:51:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.692832 23665 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 0 FOLLOWER]: Advancing to term 1
23:51:50.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.692855 23801 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 0 FOLLOWER]: Advancing to term 1
23:51:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.693955 23801 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 1.
23:51:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.693955 23665 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 1.
23:51:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.694293 23460 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 
23:51:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.694475 23883 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 FOLLOWER]: Leader election won for term 1
23:51:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.694736 23883 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 LEADER]: Becoming Leader. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Running, Role: LEADER
23:51:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.695041 23883 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:50.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.700724 23198 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 reported cstate change: term changed from 0 to 1, leader changed from <none> to 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65). New cstate: current_term: 1 leader_uuid: "63bc8b354cf246ea9511efd387cf8a55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:51:50.721 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 17075ca59c4040bcba28f9134ccae5d7
23:51:50.742 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:50.748 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:50.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=null) after lookup
23:51:50.769 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.770991 23576 tablet.cc:2406] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:50.774 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] Successfully connected to peer
23:51:50.838 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x15bbedbc, L:/127.0.0.1:52416 - R:/127.22.157.65:39389] using TOKEN/null
23:51:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.846503 23801 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Refusing update from remote peer 63bc8b354cf246ea9511efd387cf8a55: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.846655 23665 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 1 FOLLOWER]: Refusing update from remote peer 63bc8b354cf246ea9511efd387cf8a55: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.847355 23888 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.847633 23883 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:50.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.856483 23893 mvcc.cc:204] Tried to move back new op lower bound from 7305086201218977792 to 7305086200609550336. Current Snapshot: MvccSnapshot[applied={T|T < 7305086201218977792}]
23:51:50.944 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:50.946 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.65:39389
23:51:50.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:51:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.948005 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23445
23:51:50.960 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] handling channelInactive
23:51:50.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:50.960 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state READY due to: connection closed
23:51:50.961 [DEBUG - Time-limited test] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:50.963 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:50.964 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:50.964 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:50.979 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:50.979 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:50.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [3ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@430722142(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@35cb8b1d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@237b974a -> wakeup thread Time-limited test)) after lookup
23:51:50.981 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:50.983 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:50.983 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:50.983 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:50.999 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:50.999 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:51.000 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [3ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [19ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [22ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [22ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@430722142(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@35cb8b1d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@237b974a -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030))) after lookup
23:51:51.001 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:51.003 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.003 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.004 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.039 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:51.039 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:51.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [3ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [19ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [22ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [22ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [39ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [43ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [77ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@430722142(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@35cb8b1d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@237b974a -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506))) after lookup
23:51:51.041 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:51.043 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.043 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.043 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.119 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:51.119 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:51.120 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=159), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [3ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [19ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [22ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [22ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [39ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [43ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [77ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [79ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [82ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [82ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [157ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@430722142(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@35cb8b1d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506) -> (continuation of Deferred@847528775 after retry RPC@1169732026), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@237b974a -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506) -> (continuation of Deferred@847528775 after retry RPC@1169732026))) after lookup
23:51:51.121 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:51.122 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.122 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.122 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.179 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:51.179 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:51.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=219), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [3ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [19ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [22ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [22ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [39ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [43ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [77ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [79ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [82ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [82ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [157ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [159ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [161ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [161ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [214ms] refreshing cache from master, [214ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [217ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@430722142(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@35cb8b1d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506) -> (continuation of Deferred@847528775 after retry RPC@1169732026) -> (continuation of Deferred@562634584 after retry RPC@771985642), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@237b974a -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506) -> (continuation of Deferred@847528775 after retry RPC@1169732026) -> (continuation of Deferred@562634584 after retry RPC@771985642))) after lookup
23:51:51.180 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:51.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:51.182 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.401293 23908 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:51:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.401486 23908 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.401969 23909 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:51:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.402088 23909 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.402760 23908 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.402984 23909 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.403625 23601 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:51:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.403957 23737 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:51:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.404663 23601 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:51:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.405349 23737 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:51:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.407256 23665 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613" is_pre_election: true
23:51:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.407442 23665 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 1.
23:51:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.407799 23734 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:51:52.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.408053 23909 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Leader pre-election won for term 2
23:51:52.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.408123 23909 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Starting leader election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:51:52.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.408165 23909 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 1 FOLLOWER]: Advancing to term 2
23:51:52.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.408224 23801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc" is_pre_election: true
23:51:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.409077 23909 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.409349 23801 raft_consensus.cc:2393] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate acfcc37b94ba4725bc4d48a97a10e613 in current term 2: Already voted for candidate 8b14a904198a4e7fab4c5c987d1627dc in this term.
23:51:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.409539 23909 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 2 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:52.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.409965 23665 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:51:52.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.410076 23601 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc
23:51:52.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.410120 23665 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 1 FOLLOWER]: Advancing to term 2
23:51:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.410974 23665 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 2.
23:51:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.411090 23908 raft_consensus.cc:2749] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:51:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.411345 23734 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 
23:51:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.411535 23909 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Leader election won for term 2
23:51:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.411728 23909 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 LEADER]: Becoming Leader. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Running, Role: LEADER
23:51:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.411733 23737 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:51:52.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.412070 23909 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:52.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.414719 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc reported cstate change: term changed from 1 to 2, leader changed from 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65) to 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67). New cstate: current_term: 2 leader_uuid: "8b14a904198a4e7fab4c5c987d1627dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: HEALTHY } } }
23:51:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.831401 23737 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc -> Peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Couldn't send request to peer 63bc8b354cf246ea9511efd387cf8a55. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:51:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.915720 23665 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Refusing update from remote peer 8b14a904198a4e7fab4c5c987d1627dc: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
23:51:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.916386 23909 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
23:51:52.978 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:52.979 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:52.979 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2018), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [3ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [19ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [22ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [22ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [39ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [43ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [77ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [79ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [82ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [82ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [157ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [159ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [161ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [161ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [214ms] refreshing cache from master, [214ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [217ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [219ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [221ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [221ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2014ms] refreshing cache from master, [2014ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [2017ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@430722142(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@35cb8b1d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506) -> (continuation of Deferred@847528775 after retry RPC@1169732026) -> (continuation of Deferred@562634584 after retry RPC@771985642) -> (continuation of Deferred@1840517147 after retry RPC@1593989511), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@237b974a -> wakeup thread Time-limited test -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@685261565 after retry RPC@1923987506) -> (continuation of Deferred@847528775 after retry RPC@1169732026) -> (continuation of Deferred@562634584 after retry RPC@771985642) -> (continuation of Deferred@1840517147 after retry RPC@1593989511))) after lookup
23:51:52.980 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:52.981 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] Successfully connected to peer
23:51:53.003 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8440217b, L:/127.0.0.1:34332 - R:/127.22.157.67:35111] using TOKEN/null
23:51:53.023 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.65:39389
23:51:53.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.025110 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.128787 23938 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.128996 23938 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.129040 23938 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.132479 23938 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.132588 23938 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.65
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.136418 23938 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:51:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23938
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.137485 23938 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:53.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.138581 23938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:53.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.145146 23943 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:53.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.145183 23946 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.145255 23938 server_base.cc:1061] running on GCE node
23:51:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.145160 23944 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.146209 23938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.146664 23938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.147795 23938 hybrid_clock.cc:648] HybridClock initialized: now 1783468313147782 us; error 27 us; skew 500 ppm
23:51:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.149555 23938 webserver.cc:534] Webserver started at http://127.22.157.65:40325/ using document root <none> and password file <none>
23:51:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.150071 23938 fs_manager.cc:362] Metadata directory not provided
23:51:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.150130 23938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.153309 23938 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
23:51:53.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.155055 23952 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.156066 23938 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.156184 23938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.156602 23938 fs_report.cc:389] FS layout report
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/data
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:53.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.184870 23938 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.185667 23938 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.185881 23938 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:53.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.186493 23938 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:53.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.187834 23959 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:51:53.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.191005 23938 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:51:53.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.191083 23938 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
23:51:53.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.191149 23938 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:51:53.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.192662 23938 ts_tablet_manager.cc:616] Registered 1 tablets
23:51:53.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.192730 23938 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
23:51:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.193004 23959 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap starting.
23:51:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.204308 23938 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.65:39389
23:51:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.204375 24067 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.65:39389 every 8 connection(s)
23:51:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.205446 23938 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:51:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.209107 23959 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Log is configured to *not* fsync() on all Append() calls
23:51:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.211681 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 23938
23:51:53.212 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.215570 24070 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:51:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.215822 24070 heartbeater.cc:461] Registering TS with master...
23:51:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.216449 24070 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:51:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.218128 23959 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:51:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.218566 23959 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap complete.
23:51:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.218971 24068 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:51:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.219185 24068 heartbeater.cc:461] Registering TS with master...
23:51:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.219271 23271 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:51:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.219553 24068 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:51:53.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.220091 23959 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent bootstrapping tablet: real 0.027s	user 0.013s	sys 0.012s
23:51:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.220698 23345 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:51:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.220732 24069 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:51:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.220798 24069 heartbeater.cc:461] Registering TS with master...
23:51:53.221 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.220986 24069 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:51:53.223 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.67:35111
23:51:53.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.221910 23199 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.222839 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.65:43029
23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.222759 23959 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.223147 23959 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Initialized, Role: FOLLOWER
23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.223601 23959 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.224135 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23718
23:51:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.224527 23959 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
23:51:53.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.224994 24069 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:51:53.233 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] handling channelInactive
23:51:53.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:53.233 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state READY due to: connection closed
23:51:53.234 [DEBUG - Time-limited test] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:53.235 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.235 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.235 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:53.259 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:53.259 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test)) after lookup
23:51:53.260 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:53.261 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.261 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.261 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:53.278 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:53.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [25ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [27ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [27ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800))) after lookup
23:51:53.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:53.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.280 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.317 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:53.318 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:53.318 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [25ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [27ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [27ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [45ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [46ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114))) after lookup
23:51:53.319 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:53.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.320 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.320 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.338 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:53.338 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:53.339 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=105), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [25ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [27ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [27ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [45ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [46ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [84ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [86ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [86ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [104ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442))) after lookup
23:51:53.339 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:53.341 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.341 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.341 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:53.399 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:53.399 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=165), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [25ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [27ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [27ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [45ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [46ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [84ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [86ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [86ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [104ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [105ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [107ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [107ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [164ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442) -> (continuation of Deferred@2074438500 after retry RPC@1811137173), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442) -> (continuation of Deferred@2074438500 after retry RPC@1811137173))) after lookup
23:51:53.400 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:53.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:53.402 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:54.058 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:54.058 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:54.059 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=824), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [25ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [27ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [27ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [45ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [46ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [84ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [86ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [86ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [104ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [105ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [107ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [107ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [164ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [166ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [168ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [168ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [820ms] refreshing cache from master, [821ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [823ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442) -> (continuation of Deferred@2074438500 after retry RPC@1811137173) -> (continuation of Deferred@541448404 after retry RPC@71392203), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442) -> (continuation of Deferred@2074438500 after retry RPC@1811137173) -> (continuation of Deferred@541448404 after retry RPC@71392203))) after lookup
23:51:54.059 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:51:54.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:54.060 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:54.061 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:51:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.522943 24081 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 8b14a904198a4e7fab4c5c987d1627dc)
23:51:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.523098 24081 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.523710 24081 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.525233 23601 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:51:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.528638 24021 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55" is_pre_election: true
23:51:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.528900 24021 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate acfcc37b94ba4725bc4d48a97a10e613 in term 1.
23:51:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.529547 23601 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 8b14a904198a4e7fab4c5c987d1627dc
23:51:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.529752 24081 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Leader pre-election won for term 3
23:51:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.529825 24081 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 8b14a904198a4e7fab4c5c987d1627dc)
23:51:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.529853 24081 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 2 FOLLOWER]: Advancing to term 3
23:51:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.530615 24081 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.530946 24081 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 3 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.531400 24021 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:51:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.531515 24021 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 1 FOLLOWER]: Advancing to term 3
23:51:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.532267 23601 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:51:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.533068 24021 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate acfcc37b94ba4725bc4d48a97a10e613 in term 3.
23:51:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.533362 23601 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 8b14a904198a4e7fab4c5c987d1627dc
23:51:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.533622 24081 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 FOLLOWER]: Leader election won for term 3
23:51:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.533811 24081 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 LEADER]: Becoming Leader. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Running, Role: LEADER
23:51:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.534099 24081 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.536705 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 reported cstate change: term changed from 2 to 3, leader changed from 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67) to acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66). New cstate: current_term: 3 leader_uuid: "acfcc37b94ba4725bc4d48a97a10e613" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:51:54.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.945220 24021 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Refusing update from remote peer acfcc37b94ba4725bc4d48a97a10e613: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
23:51:54.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.945924 24081 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
23:51:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.955018 23601 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 -> Peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Couldn't send request to peer 8b14a904198a4e7fab4c5c987d1627dc. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:51:55.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:55.398 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:55.399 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2164), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [1ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [24ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [25ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [27ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [27ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [45ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [46ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [84ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [86ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [86ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [104ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [105ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [107ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [107ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [164ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [166ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [168ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [168ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [820ms] refreshing cache from master, [821ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [823ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [825ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [827ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [827ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2160ms] refreshing cache from master, [2161ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [2163ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1239857760(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@4650cba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442) -> (continuation of Deferred@2074438500 after retry RPC@1811137173) -> (continuation of Deferred@541448404 after retry RPC@71392203) -> (continuation of Deferred@1802087145 after retry RPC@1702374220), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@76a415e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1477554146 after retry RPC@248287800) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@1648190229 after retry RPC@488126442) -> (continuation of Deferred@2074438500 after retry RPC@1811137173) -> (continuation of Deferred@541448404 after retry RPC@71392203) -> (continuation of Deferred@1802087145 after retry RPC@1702374220))) after lookup
23:51:55.399 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:55.400 [DEBUG - kudu-nio-2] (Connection.java:757) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] Successfully connected to peer
23:51:55.423 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x3b3efba1, L:/127.0.0.1:57688 - R:/127.22.157.66:37793] using TOKEN/null
23:51:55.448 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.67:35111
23:51:55.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:51:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.449724 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:35111
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45069
23:51:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:51:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:55.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.566844 24111 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:55.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.567082 24111 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:55.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.567129 24111 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.570791 24111 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.570905 24111 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.67
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.575042 24111 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:35111
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:51:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45069
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24111
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.576315 24111 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.577517 24111 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.584110 24117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.584111 24116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.584257 24111 server_base.cc:1061] running on GCE node
23:51:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.584111 24119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.585062 24111 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.585618 24111 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.586760 24111 hybrid_clock.cc:648] HybridClock initialized: now 1783468315586744 us; error 36 us; skew 500 ppm
23:51:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.589005 24111 webserver.cc:534] Webserver started at http://127.22.157.67:45069/ using document root <none> and password file <none>
23:51:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.589531 24111 fs_manager.cc:362] Metadata directory not provided
23:51:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.589597 24111 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.592813 24111 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
23:51:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.594609 24125 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.595588 24111 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.595706 24111 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.596145 24111 fs_report.cc:389] FS layout report
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/data
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:55.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.624766 24111 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:55.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.625522 24111 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:55.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.625708 24111 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:55.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.626318 24111 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:55.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.627622 24132 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:51:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.630774 24111 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:51:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.630851 24111 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
23:51:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.630911 24111 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:51:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.632355 24111 ts_tablet_manager.cc:616] Registered 1 tablets
23:51:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.632422 24111 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
23:51:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.632689 24132 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap starting.
23:51:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.643776 24111 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.67:35111
23:51:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.643851 24241 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.67:35111 every 8 connection(s)
23:51:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.644836 24111 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:51:55.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:55.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.646647 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 24111
23:51:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.648660 24132 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Log is configured to *not* fsync() on all Append() calls
23:51:55.656 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:55.657 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.66:37793
23:51:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.655718 24243 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:51:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.655939 24243 heartbeater.cc:461] Registering TS with master...
23:51:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.656441 24243 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:51:55.658 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:51:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.658447 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23583
23:51:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.660452 23197 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.660638 24244 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:51:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.660720 24244 heartbeater.cc:461] Registering TS with master...
23:51:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.660931 24244 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:51:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.661383 24242 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.661391 23197 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.67:38395
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.661459 24242 heartbeater.cc:461] Registering TS with master...
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.661616 24242 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.661729 24132 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.661839 23271 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.662102 24132 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap complete.
23:51:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.662793 23345 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.663378 24132 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent bootstrapping tablet: real 0.031s	user 0.011s	sys 0.014s
23:51:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.665347 24132 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.665688 24132 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Initialized, Role: FOLLOWER
23:51:55.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.666145 24132 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.666821 24243 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:51:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.667057 24132 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent starting tablet: real 0.003s	user 0.003s	sys 0.000s
23:51:55.668 [DEBUG - kudu-nio-2] (Connection.java:239) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] handling channelInactive
23:51:55.668 [DEBUG - kudu-nio-2] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state READY due to: connection closed
23:51:55.669 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:55.670 [DEBUG - Time-limited test] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:55.671 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.671 [DEBUG - kudu-nio-3] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.671 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:55.678 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:55.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test)) after lookup
23:51:55.679 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:55.680 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.680 [DEBUG - kudu-nio-4] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.680 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:55.698 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:55.699 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [10ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [10ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979))) after lookup
23:51:55.699 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:55.701 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.701 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.739 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:55.739 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:55.739 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [10ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [10ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [31ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [31ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [68ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431))) after lookup
23:51:55.740 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:55.741 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.741 [DEBUG - kudu-nio-6] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.741 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:55.858 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:55.859 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=189), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [10ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [10ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [31ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [31ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [68ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [71ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [71ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [185ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [188ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146))) after lookup
23:51:55.859 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:55.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:55.861 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.158 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:56.158 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:56.159 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=489), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [10ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [10ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [31ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [31ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [68ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [71ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [71ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [185ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [188ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [189ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [191ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [191ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [487ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146) -> (continuation of Deferred@655465428 after retry RPC@854727327), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146) -> (continuation of Deferred@655465428 after retry RPC@854727327))) after lookup
23:51:56.159 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:56.160 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.160 [DEBUG - kudu-nio-0] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.160 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:56.658 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:51:56.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=988), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [10ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [10ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [31ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [31ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [68ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [71ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [71ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [185ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [188ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [189ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [191ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [191ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [487ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [489ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [490ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [490ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [985ms] refreshing cache from master, [985ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [987ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146) -> (continuation of Deferred@655465428 after retry RPC@854727327) -> (continuation of Deferred@1130117574 after retry RPC@253668070), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146) -> (continuation of Deferred@655465428 after retry RPC@854727327) -> (continuation of Deferred@1130117574 after retry RPC@253668070))) after lookup
23:51:56.659 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:51:56.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.660 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:51:56.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.947748 24255 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader acfcc37b94ba4725bc4d48a97a10e613)
23:51:56.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.947901 24255 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.949009 24255 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:56.949714 23953 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:51:56.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:56.950598 23953 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:51:56.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.954021 24195 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc" is_pre_election: true
23:51:56.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.954255 24195 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 2.
23:51:56.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.954959 23956 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc; no voters: acfcc37b94ba4725bc4d48a97a10e613
23:51:56.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.955178 24255 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Leader pre-election won for term 4
23:51:56.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.955236 24255 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Starting leader election (detected failure of leader acfcc37b94ba4725bc4d48a97a10e613)
23:51:56.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.955283 24255 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 3 FOLLOWER]: Advancing to term 4
23:51:56.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.956115 24255 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:56.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.956483 24255 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 4 election: Requested vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:51:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.956974 24195 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:51:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.957101 24195 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 2 FOLLOWER]: Advancing to term 4
23:51:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:56.957492 23953 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:51:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.959175 24195 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 4.
23:51:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.959491 23956 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc; no voters: acfcc37b94ba4725bc4d48a97a10e613
23:51:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.959642 24255 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 FOLLOWER]: Leader election won for term 4
23:51:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.959862 24255 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 LEADER]: Becoming Leader. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Running, Role: LEADER
23:51:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.960203 24255 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:56.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.962383 23197 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 reported cstate change: term changed from 3 to 4, leader changed from acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66) to 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65). New cstate: current_term: 4 leader_uuid: "63bc8b354cf246ea9511efd387cf8a55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:51:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:57.359303 23953 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 -> Peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Couldn't send request to peer acfcc37b94ba4725bc4d48a97a10e613. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:51:57.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.424227 24195 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Refusing update from remote peer 63bc8b354cf246ea9511efd387cf8a55: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
23:51:57.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.424996 24255 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
23:51:58.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:58.958 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:58.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3288), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [10ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [10ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [31ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [31ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [68ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [71ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [71ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [185ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [188ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [189ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [191ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [191ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [487ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [489ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [490ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [490ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [985ms] refreshing cache from master, [985ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [987ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [989ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [990ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [990ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [3285ms] refreshing cache from master, [3285ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [3287ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1921304983(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5602bec0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146) -> (continuation of Deferred@655465428 after retry RPC@854727327) -> (continuation of Deferred@1130117574 after retry RPC@253668070) -> (continuation of Deferred@703401823 after retry RPC@1407109406), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4559b742 -> wakeup thread Time-limited test -> (continuation of Deferred@1721914634 after retry RPC@1858673979) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@1028357832 after retry RPC@1416115146) -> (continuation of Deferred@655465428 after retry RPC@854727327) -> (continuation of Deferred@1130117574 after retry RPC@253668070) -> (continuation of Deferred@703401823 after retry RPC@1407109406))) after lookup
23:51:58.959 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:58.960 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] Successfully connected to peer
23:51:58.979 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xa4b95d0a, L:/127.0.0.1:40954 - R:/127.22.157.65:39389] using TOKEN/null
23:51:59.004 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.66:37793
23:51:59.004 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:51:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.005198 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:51:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:51:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:37793
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43755
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.115098 24293 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.115294 24293 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.115336 24293 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:59.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.118673 24293 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:59.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.118781 24293 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.66
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.122285 24293 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:37793
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43755
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24293
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:51:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.123325 24293 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:59.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.124416 24293 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.130540 24301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.130647 24293 server_base.cc:1061] running on GCE node
23:51:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.130550 24299 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.130545 24298 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.131062 24293 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.131583 24293 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.132753 24293 hybrid_clock.cc:648] HybridClock initialized: now 1783468319132737 us; error 30 us; skew 500 ppm
23:51:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.134431 24293 webserver.cc:534] Webserver started at http://127.22.157.66:43755/ using document root <none> and password file <none>
23:51:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.134987 24293 fs_manager.cc:362] Metadata directory not provided
23:51:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.135088 24293 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.138485 24293 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
23:51:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.140322 24307 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.141353 24293 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
23:51:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.141470 24293 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.141876 24293 fs_report.cc:389] FS layout report
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/data
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.153604 24293 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.154220 24293 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.154405 24293 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.154963 24293 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.156201 24314 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:51:59.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.159178 24293 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:51:59.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.159258 24293 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
23:51:59.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.159319 24293 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:51:59.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.160800 24293 ts_tablet_manager.cc:616] Registered 1 tablets
23:51:59.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.160897 24293 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
23:51:59.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.161255 24314 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap starting.
23:51:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.171568 24293 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.66:37793
23:51:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.171622 24422 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.66:37793 every 8 connection(s)
23:51:59.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.172577 24293 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:51:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.178931 24314 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Log is configured to *not* fsync() on all Append() calls
23:51:59.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.181789 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 24293
23:51:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.182438 24425 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:51:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.182653 24425 heartbeater.cc:461] Registering TS with master...
23:51:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.182861 24424 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:51:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.182968 24424 heartbeater.cc:461] Registering TS with master...
23:51:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.183197 24424 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:51:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.183197 24425 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:51:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.184998 23199 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.185101 23272 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:59.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.185959 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.66:41035
23:51:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.187686 24423 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:51:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.187774 24423 heartbeater.cc:461] Registering TS with master...
23:51:59.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.188167 24423 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:51:59.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.189078 23344 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:51:59.190 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:59.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.194904 24314 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:51:59.197 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.65:39389
23:51:59.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:51:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.195410 24314 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap complete.
23:51:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.196835 24314 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent bootstrapping tablet: real 0.036s	user 0.018s	sys 0.012s
23:51:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.197974 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23938
23:51:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.199832 24314 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.200284 24314 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Initialized, Role: FOLLOWER
23:51:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.200850 24314 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:51:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.201529 24424 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:51:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.201756 24314 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
23:51:59.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:59.212 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] handling channelInactive
23:51:59.212 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state READY due to: connection closed
23:51:59.216 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:59.217 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.217 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.218 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:59.238 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:59.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [6ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [25ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@2049293028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5f961a87 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34d306c4 -> wakeup thread Time-limited test)) after lookup
23:51:59.239 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:59.240 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.240 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.241 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:59.259 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:59.259 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [6ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [25ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [27ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [29ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [29ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [46ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@2049293028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5f961a87 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34d306c4 -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144))) after lookup
23:51:59.260 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:59.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.261 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.261 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:59.279 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:59.279 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [6ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [25ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [27ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [29ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [29ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [46ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [47ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [49ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [49ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [66ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@2049293028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5f961a87 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34d306c4 -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445))) after lookup
23:51:59.280 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:59.281 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.281 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.281 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:51:59.438 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:51:59.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=226), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [6ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [25ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [27ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [29ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [29ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [46ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [47ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [49ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [49ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [66ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [67ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [69ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [69ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [226ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@2049293028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5f961a87 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445) -> (continuation of Deferred@85308700 after retry RPC@1089430211), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34d306c4 -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445) -> (continuation of Deferred@85308700 after retry RPC@1089430211))) after lookup
23:51:59.439 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:51:59.440 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.440 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:51:59.441 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:00.058 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:00.059 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:00.059 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=847), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [6ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [25ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [27ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [29ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [29ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [46ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [47ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [49ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [49ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [66ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [67ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [69ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [69ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [226ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [227ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [229ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [229ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [843ms] refreshing cache from master, [843ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [845ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@2049293028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5f961a87 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445) -> (continuation of Deferred@85308700 after retry RPC@1089430211) -> (continuation of Deferred@2145784760 after retry RPC@1143250941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34d306c4 -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445) -> (continuation of Deferred@85308700 after retry RPC@1089430211) -> (continuation of Deferred@2145784760 after retry RPC@1143250941))) after lookup
23:52:00.060 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:00.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:00.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:00.061 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.499886 24436 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:52:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.500061 24436 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:00.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.501319 24436 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:00.502085 24129 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:00.502940 24129 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.505995 24377 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613" is_pre_election: true
23:52:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.506232 24377 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 3.
23:52:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.506860 24126 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:52:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.507133 24436 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Leader pre-election won for term 5
23:52:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.507190 24436 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Starting leader election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:52:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.507218 24436 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 4 FOLLOWER]: Advancing to term 5
23:52:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.508023 24436 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.508324 24436 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 5 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.508719 24377 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:52:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.508846 24377 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 3 FOLLOWER]: Advancing to term 5
23:52:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:00.509371 24129 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.510607 24377 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 5.
23:52:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.510944 24126 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:52:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.511101 24436 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Leader election won for term 5
23:52:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.511284 24436 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 LEADER]: Becoming Leader. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Running, Role: LEADER
23:52:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.511638 24436 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.513846 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc reported cstate change: term changed from 4 to 5, leader changed from 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65) to 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67). New cstate: current_term: 5 leader_uuid: "8b14a904198a4e7fab4c5c987d1627dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: HEALTHY } } }
23:52:01.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.050050 24377 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Refusing update from remote peer 8b14a904198a4e7fab4c5c987d1627dc: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
23:52:01.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.050907 24454 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
23:52:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.059121 24129 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc -> Peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Couldn't send request to peer 63bc8b354cf246ea9511efd387cf8a55. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:01.198 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:01.198 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:01.199 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1987), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [6ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [25ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [27ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [29ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [29ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [46ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [47ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [49ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [49ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [66ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [67ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [69ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [69ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [226ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [227ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [229ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [229ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [843ms] refreshing cache from master, [843ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [845ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [847ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [849ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [849ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [1983ms] refreshing cache from master, [1983ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [1986ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@2049293028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@5f961a87 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445) -> (continuation of Deferred@85308700 after retry RPC@1089430211) -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@2145804364 after retry RPC@574187510), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34d306c4 -> wakeup thread Time-limited test -> (continuation of Deferred@702862734 after retry RPC@86840144) -> (continuation of Deferred@1986132618 after retry RPC@465293445) -> (continuation of Deferred@85308700 after retry RPC@1089430211) -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@2145804364 after retry RPC@574187510))) after lookup
23:52:01.199 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:01.201 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] Successfully connected to peer
23:52:01.214 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x192f70f3, L:/127.0.0.1:55312 - R:/127.22.157.67:35111] using TOKEN/null
23:52:01.231 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.65:39389
23:52:01.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.232811 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.335932 24469 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.336171 24469 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.336232 24469 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.339772 24469 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:01.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.339921 24469 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.65
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.343859 24469 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24469
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.345355 24469 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.346575 24469 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.353031 24477 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.353055 24474 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.353034 24475 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.353099 24469 server_base.cc:1061] running on GCE node
23:52:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.353516 24469 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:01.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.354017 24469 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:01.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.355187 24469 hybrid_clock.cc:648] HybridClock initialized: now 1783468321355174 us; error 26 us; skew 500 ppm
23:52:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.357000 24469 webserver.cc:534] Webserver started at http://127.22.157.65:40325/ using document root <none> and password file <none>
23:52:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.357606 24469 fs_manager.cc:362] Metadata directory not provided
23:52:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.357692 24469 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:01.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.360997 24469 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
23:52:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.362859 24483 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.363819 24469 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.363940 24469 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.364364 24469 fs_report.cc:389] FS layout report
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/data
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.376215 24469 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.376926 24469 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.377112 24469 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.377640 24469 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.378901 24490 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.382105 24469 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.382176 24469 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
23:52:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.382236 24469 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.383653 24469 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.383713 24469 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
23:52:01.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.384017 24490 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap starting.
23:52:01.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.394479 24469 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.65:39389
23:52:01.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.394497 24598 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.65:39389 every 8 connection(s)
23:52:01.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.395588 24469 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:01.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.398905 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 24469
23:52:01.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.400072 24490 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Log is configured to *not* fsync() on all Append() calls
23:52:01.408 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:01.408 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.67:35111
23:52:01.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.409204 24599 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.409224 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 24111
23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.409447 24599 heartbeater.cc:461] Registering TS with master...
23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.410039 24599 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.410918 24600 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.411002 24600 heartbeater.cc:461] Registering TS with master...
23:52:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.411211 24600 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.412390 23345 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.413101 24601 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.413179 24601 heartbeater.cc:461] Registering TS with master...
23:52:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.413355 23199 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.413381 24601 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.414263 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.65:43789
23:52:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.414570 23272 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:01.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:01.418 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] handling channelInactive
23:52:01.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state READY due to: connection closed
23:52:01.418 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: connection closed
23:52:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.419895 24490 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.420339 24490 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap complete.
23:52:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.421440 24490 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent bootstrapping tablet: real 0.038s	user 0.021s	sys 0.010s
23:52:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.423594 24490 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.423962 24490 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Initialized, Role: FOLLOWER
23:52:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.424413 24490 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.425127 24600 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:52:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.425369 24490 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
23:52:01.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:01.438 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:01.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [0ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [20ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1399210648(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@3a8999d5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2459557c -> wakeup thread Time-limited test)) after lookup
23:52:01.439 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:01.440 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.440 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.440 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:01.458 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:01.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [0ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [20ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [21ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [22ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [23ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1399210648(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@3a8999d5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2459557c -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404))) after lookup
23:52:01.459 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:01.460 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.460 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.460 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:01.498 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:01.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [0ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [20ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [21ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [22ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [23ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [41ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [42ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [42ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1399210648(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@3a8999d5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2459557c -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384))) after lookup
23:52:01.499 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:01.500 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.500 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.500 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:01.558 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:01.559 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=140), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [0ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [20ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [21ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [22ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [23ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [41ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [42ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [42ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [81ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [82ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [82ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [140ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1399210648(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@3a8999d5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2459557c -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909))) after lookup
23:52:01.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:01.560 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.560 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.561 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.938 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:01.938 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:01.939 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=520), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [0ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [20ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [21ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [22ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [23ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [41ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [42ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [42ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [81ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [82ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [82ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [140ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [141ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [142ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [143ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [517ms] refreshing cache from master, [517ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [520ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1399210648(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@3a8999d5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2459557c -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773))) after lookup
23:52:01.939 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:01.940 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.940 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:01.941 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.731865 24613 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 8b14a904198a4e7fab4c5c987d1627dc)
23:52:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.732044 24613 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.733144 24613 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:02.734258 24311 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:52:02.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:02.735209 24311 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:52:02.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.738021 24553 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55" is_pre_election: true
23:52:02.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.738343 24553 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate acfcc37b94ba4725bc4d48a97a10e613 in term 4.
23:52:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.739078 24311 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 8b14a904198a4e7fab4c5c987d1627dc
23:52:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.739351 24613 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Leader pre-election won for term 6
23:52:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.739477 24613 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 8b14a904198a4e7fab4c5c987d1627dc)
23:52:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.739517 24613 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 5 FOLLOWER]: Advancing to term 6
23:52:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.740322 24613 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.740633 24613 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 6 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.741109 24553 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:52:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.741262 24553 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 4 FOLLOWER]: Advancing to term 6
23:52:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:02.741966 24311 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:52:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.743088 24553 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate acfcc37b94ba4725bc4d48a97a10e613 in term 6.
23:52:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.743443 24311 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 8b14a904198a4e7fab4c5c987d1627dc
23:52:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.743595 24613 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 FOLLOWER]: Leader election won for term 6
23:52:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.743901 24613 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 LEADER]: Becoming Leader. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Running, Role: LEADER
23:52:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.744241 24613 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:02.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.746629 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 reported cstate change: term changed from 5 to 6, leader changed from 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67) to acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66). New cstate: current_term: 6 leader_uuid: "acfcc37b94ba4725bc4d48a97a10e613" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:52:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.226187 24311 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 -> Peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Couldn't send request to peer 8b14a904198a4e7fab4c5c987d1627dc. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:03.338 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:03.338 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:03.339 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1920), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [0ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [20ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [21ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [22ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [23ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [41ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [42ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [42ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [81ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [82ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [82ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [140ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [141ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [142ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [143ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [517ms] refreshing cache from master, [517ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [520ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [521ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [523ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [523ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [1917ms] refreshing cache from master, [1917ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [1920ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1399210648(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@3a8999d5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773) -> (continuation of Deferred@558970929 after retry RPC@1405117011), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2459557c -> wakeup thread Time-limited test -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773) -> (continuation of Deferred@558970929 after retry RPC@1405117011))) after lookup
23:52:03.339 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:03.340 [DEBUG - kudu-nio-7] (Connection.java:757) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] Successfully connected to peer
23:52:03.363 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xdd8c03ab, L:/127.0.0.1:57786 - R:/127.22.157.66:37793] using TOKEN/null
23:52:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.365237 24553 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Refusing update from remote peer acfcc37b94ba4725bc4d48a97a10e613: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
23:52:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.365978 24613 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
23:52:03.396 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.67:35111
23:52:03.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.398634 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:35111
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45069
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:52:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.524299 24641 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.524597 24641 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.524657 24641 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.529338 24641 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.529491 24641 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.67
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.533458 24641 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:35111
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45069
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24641
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.534533 24641 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.535959 24641 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.542290 24649 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.542291 24646 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:03.542457 24647 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.542526 24641 server_base.cc:1061] running on GCE node
23:52:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.543135 24641 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.543756 24641 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.544926 24641 hybrid_clock.cc:648] HybridClock initialized: now 1783468323544899 us; error 39 us; skew 500 ppm
23:52:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.546828 24641 webserver.cc:534] Webserver started at http://127.22.157.67:45069/ using document root <none> and password file <none>
23:52:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.547472 24641 fs_manager.cc:362] Metadata directory not provided
23:52:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.547554 24641 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.550733 24641 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
23:52:03.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.552675 24655 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.553586 24641 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.553710 24641 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.554138 24641 fs_report.cc:389] FS layout report
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/data
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:03.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.566885 24641 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:03.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.567595 24641 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:03.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.567782 24641 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:03.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.568434 24641 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:03.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.569808 24663 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:03.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.573133 24641 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:03.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.573210 24641 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
23:52:03.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.573288 24641 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.574807 24641 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.574878 24641 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
23:52:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.575196 24663 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap starting.
23:52:03.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.586498 24641 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.67:35111
23:52:03.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.586526 24770 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.67:35111 every 8 connection(s)
23:52:03.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.587584 24641 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:52:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.590670 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 24641
23:52:03.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.592338 24663 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Log is configured to *not* fsync() on all Append() calls
23:52:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.599242 24771 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.599506 24771 heartbeater.cc:461] Registering TS with master...
23:52:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.600033 24771 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.601884 23345 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.603284 24773 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.603394 24773 heartbeater.cc:461] Registering TS with master...
23:52:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.603593 24773 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.604918 23272 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.605155 24772 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.605239 24772 heartbeater.cc:461] Registering TS with master...
23:52:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.605429 24772 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.606241 23199 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.607213 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.67:41469
23:52:03.607 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:03.608 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.66:37793
23:52:03.608 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.608752 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 24293
23:52:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.614557 24663 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.615053 24663 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap complete.
23:52:03.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:03.616 [DEBUG - kudu-nio-7] (Connection.java:239) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] handling channelInactive
23:52:03.616 [DEBUG - kudu-nio-7] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state READY due to: connection closed
23:52:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.617539 24663 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent bootstrapping tablet: real 0.043s	user 0.024s	sys 0.010s
23:52:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.620103 24663 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.620481 24663 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Initialized, Role: FOLLOWER
23:52:03.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.621004 24663 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.621722 24772 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:52:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:03.621963 24663 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
23:52:03.635 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:03.637 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.637 [DEBUG - kudu-nio-0] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.637 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:03.658 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:03.659 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [21ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [21ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1700035387(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@2d2d2614 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c5bd7a8 -> wakeup thread Time-limited test)) after lookup
23:52:03.659 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:03.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.661 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:03.678 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:03.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [21ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [21ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [43ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [44ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [45ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1700035387(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@2d2d2614 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c5bd7a8 -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545))) after lookup
23:52:03.679 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:03.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.680 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.738 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:03.738 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:03.739 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [21ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [21ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [43ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [44ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [45ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [62ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [64ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [64ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1700035387(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@2d2d2614 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c5bd7a8 -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382))) after lookup
23:52:03.740 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:03.741 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.741 [DEBUG - kudu-nio-3] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.741 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.798 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:03.798 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:03.799 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=182), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [21ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [21ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [43ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [44ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [45ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [62ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [64ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [64ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [123ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [125ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [125ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [182ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1700035387(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@2d2d2614 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382) -> (continuation of Deferred@1549358165 after retry RPC@113198797), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c5bd7a8 -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382) -> (continuation of Deferred@1549358165 after retry RPC@113198797))) after lookup
23:52:03.800 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:03.801 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.801 [DEBUG - kudu-nio-4] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:03.801 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:04.379 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:04.379 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:04.380 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=764), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [21ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [21ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [43ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [44ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [45ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [62ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [64ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [64ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [123ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [125ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [125ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [182ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [183ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [185ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [185ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [759ms] refreshing cache from master, [760ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [763ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1700035387(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@2d2d2614 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382) -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c5bd7a8 -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382) -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924))) after lookup
23:52:04.381 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:04.382 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:04.382 [DEBUG - kudu-nio-5] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:04.382 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.813290 24784 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:52:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.813544 24784 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:04.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.814700 24784 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:04.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.815539 24656 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:04.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.816605 24656 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.818943 24553 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55" is_pre_election: true
23:52:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.819496 24660 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc; no voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613
23:52:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.819772 24784 raft_consensus.cc:2749] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
23:52:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.894905 24789 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader acfcc37b94ba4725bc4d48a97a10e613)
23:52:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.895104 24789 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.896402 24789 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.897495 24484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.898483 24484 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.901530 24725 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc" is_pre_election: true
23:52:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.901842 24725 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 5.
23:52:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.902616 24487 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc; no voters: acfcc37b94ba4725bc4d48a97a10e613
23:52:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.902858 24789 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Leader pre-election won for term 7
23:52:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.902930 24789 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Starting leader election (detected failure of leader acfcc37b94ba4725bc4d48a97a10e613)
23:52:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.902976 24789 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 6 FOLLOWER]: Advancing to term 7
23:52:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.903858 24789 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.904251 24789 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 7 election: Requested vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.904726 24725 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:52:04.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.904881 24725 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 5 FOLLOWER]: Advancing to term 7
23:52:04.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.905360 24484 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:04.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.906699 24725 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 7.
23:52:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.907038 24487 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc; no voters: acfcc37b94ba4725bc4d48a97a10e613
23:52:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.907210 24789 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 FOLLOWER]: Leader election won for term 7
23:52:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.907502 24789 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 LEADER]: Becoming Leader. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Running, Role: LEADER
23:52:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.907835 24789 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:04.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:04.910312 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 reported cstate change: term changed from 6 to 7, leader changed from acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66) to 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65). New cstate: current_term: 7 leader_uuid: "63bc8b354cf246ea9511efd387cf8a55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:52:05.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.370014 24484 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 -> Peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Couldn't send request to peer acfcc37b94ba4725bc4d48a97a10e613. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:05.428753 24725 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Refusing update from remote peer 63bc8b354cf246ea9511efd387cf8a55: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
23:52:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:05.429554 24789 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
23:52:06.638 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:06.638 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:06.639 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=3022), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [21ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [21ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [43ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [44ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [45ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [62ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [64ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [64ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [123ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [125ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [125ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [182ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [183ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [185ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [185ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [759ms] refreshing cache from master, [760ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [763ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [765ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [766ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [766ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [3019ms] refreshing cache from master, [3019ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [3022ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1700035387(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@2d2d2614 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382) -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c5bd7a8 -> wakeup thread Time-limited test -> (continuation of Deferred@1850444991 after retry RPC@1015017545) -> (continuation of Deferred@296698482 after retry RPC@1376851382) -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818))) after lookup
23:52:06.639 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:06.640 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] Successfully connected to peer
23:52:06.661 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x2d9a2a39, L:/127.0.0.1:51126 - R:/127.22.157.65:39389] using TOKEN/null
23:52:06.677 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.66:37793
23:52:06.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.678649 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:37793
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43755
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.786396 24825 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.786633 24825 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.786679 24825 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.790134 24825 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.790244 24825 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.66
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.794160 24825 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:37793
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43755
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24825
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.795420 24825 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.796515 24825 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.802991 24830 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.803072 24825 server_base.cc:1061] running on GCE node
23:52:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.802999 24833 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.802999 24831 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.803942 24825 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.804529 24825 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.805712 24825 hybrid_clock.cc:648] HybridClock initialized: now 1783468326805695 us; error 35 us; skew 500 ppm
23:52:06.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.807665 24825 webserver.cc:534] Webserver started at http://127.22.157.66:43755/ using document root <none> and password file <none>
23:52:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.808251 24825 fs_manager.cc:362] Metadata directory not provided
23:52:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.808347 24825 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:06.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.811781 24825 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.004s	sys 0.000s
23:52:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.813781 24839 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.814724 24825 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.814860 24825 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.815366 24825 fs_report.cc:389] FS layout report
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/data
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.841271 24825 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.842047 24825 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.842268 24825 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:06.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.842912 24825 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:06.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.844353 24847 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.847636 24825 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.847713 24825 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
23:52:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.847779 24825 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.849232 24825 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.849311 24825 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
23:52:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.849591 24847 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap starting.
23:52:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.860671 24825 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.66:37793
23:52:06.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.860740 24956 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.66:37793 every 8 connection(s)
23:52:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.861999 24825 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:52:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.866650 24847 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Log is configured to *not* fsync() on all Append() calls
23:52:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.867556 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 24825
23:52:06.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.874794 24957 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.875042 24957 heartbeater.cc:461] Registering TS with master...
23:52:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.875706 24957 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.877781 23345 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:06.878 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:06.879 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.65:39389
23:52:06.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.880501 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 24469
23:52:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.881692 24958 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.881794 24958 heartbeater.cc:461] Registering TS with master...
23:52:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.882030 24958 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.882894 23199 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.883822 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.66:40243
23:52:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.886210 24959 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.886307 24959 heartbeater.cc:461] Registering TS with master...
23:52:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.886497 24959 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:06.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.887461 23271 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:06.889 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:06.889 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] handling channelInactive
23:52:06.889 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state READY due to: connection closed
23:52:06.889 [DEBUG - Time-limited test] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:06.890 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.891 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.891 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.892050 24847 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.892460 24847 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap complete.
23:52:06.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.893535 24847 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent bootstrapping tablet: real 0.044s	user 0.030s	sys 0.008s
23:52:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.895716 24847 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.896093 24847 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Initialized, Role: FOLLOWER
23:52:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.896674 24847 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.897373 24958 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:52:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:06.897686 24847 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
23:52:06.898 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:06.899 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:06.899 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test)) after lookup
23:52:06.900 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:06.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.901 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.918 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:06.918 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:06.918 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [12ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [12ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650))) after lookup
23:52:06.919 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:06.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.920 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.938 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:06.938 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:06.939 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [12ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [12ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [31ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [31ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [49ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043))) after lookup
23:52:06.939 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:06.940 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.940 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.940 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:06.958 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:06.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [12ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [12ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [31ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [31ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [49ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [51ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [52ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [69ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874))) after lookup
23:52:06.959 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:06.960 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.960 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:06.961 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:07.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:07.378 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:07.379 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=489), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [12ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [12ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [31ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [31ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [49ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [51ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [52ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [69ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [71ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [486ms] refreshing cache from master, [487ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [489ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115))) after lookup
23:52:07.379 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:07.380 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:07.380 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:07.380 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:07.457 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:07.458 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:07.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=568), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [12ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [12ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [31ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [31ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [49ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [51ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [52ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [69ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [71ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [486ms] refreshing cache from master, [487ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [489ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [490ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [491ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [491ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [566ms] refreshing cache from master, [566ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [568ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021))) after lookup
23:52:07.459 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:07.460 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:07.460 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:07.460 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:08.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.177341 24970 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:52:08.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.177496 24970 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.177997 24970 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:08.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.179327 24660 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:08.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.182869 24910 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613" is_pre_election: true
23:52:08.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.183125 24910 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 6.
23:52:08.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.183820 24656 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:52:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.184056 24970 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Leader pre-election won for term 8
23:52:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.184115 24970 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Starting leader election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:52:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.184144 24970 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 7 FOLLOWER]: Advancing to term 8
23:52:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.184918 24970 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.185374 24970 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 8 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.185804 24910 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:52:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.185945 24910 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 6 FOLLOWER]: Advancing to term 8
23:52:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.186480 24660 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:08.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.187798 24910 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 8.
23:52:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.188127 24656 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:52:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.188297 24970 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 FOLLOWER]: Leader election won for term 8
23:52:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.188570 24970 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 LEADER]: Becoming Leader. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Running, Role: LEADER
23:52:08.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.188905 24970 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:08.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.191257 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc reported cstate change: term changed from 7 to 8, leader changed from 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65) to 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67). New cstate: current_term: 8 leader_uuid: "8b14a904198a4e7fab4c5c987d1627dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: HEALTHY } } }
23:52:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.589313 24660 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc -> Peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Couldn't send request to peer 63bc8b354cf246ea9511efd387cf8a55. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.814069 24910 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Refusing update from remote peer 8b14a904198a4e7fab4c5c987d1627dc: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
23:52:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.814817 24970 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
23:52:09.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:09.882583 24660 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111) [suppressed 5 similar messages]
23:52:10.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:10.558 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:10.559 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3669), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [12ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [12ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [28ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [31ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [31ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [49ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [51ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [52ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [69ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [70ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [71ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [486ms] refreshing cache from master, [487ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [489ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [490ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [491ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [491ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [566ms] refreshing cache from master, [566ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [568ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [569ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [571ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [571ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [3666ms] refreshing cache from master, [3666ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [3668ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@818815713(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@45aa997e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@329d815f -> wakeup thread Time-limited test -> (continuation of Deferred@1918831615 after retry RPC@819385650) -> (continuation of Deferred@731745063 after retry RPC@769963043) -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016))) after lookup
23:52:10.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:10.561 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] Successfully connected to peer
23:52:10.577 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xfbc77f78, L:/127.0.0.1:47508 - R:/127.22.157.67:35111] using TOKEN/null
23:52:10.598 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.65:39389
23:52:10.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.599840 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.705044 25008 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.705345 25008 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.705408 25008 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.709028 25008 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.709138 25008 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.65
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.713049 25008 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25008
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.714257 25008 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.715409 25008 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.722028 25017 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.722034 25013 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.722090 25008 server_base.cc:1061] running on GCE node
23:52:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.722028 25015 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:10.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.722988 25008 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:10.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.723570 25008 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.724762 25008 hybrid_clock.cc:648] HybridClock initialized: now 1783468330724745 us; error 30 us; skew 500 ppm
23:52:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.726584 25008 webserver.cc:534] Webserver started at http://127.22.157.65:40325/ using document root <none> and password file <none>
23:52:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.727197 25008 fs_manager.cc:362] Metadata directory not provided
23:52:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.727288 25008 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:10.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.730504 25008 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.001s
23:52:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.732503 25023 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.733436 25008 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.733564 25008 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.734014 25008 fs_report.cc:389] FS layout report
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/data
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:10.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.744145 25008 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:10.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.744807 25008 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:10.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.745008 25008 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:10.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.745589 25008 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:10.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.746876 25030 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:10.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.750133 25008 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:10.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.750221 25008 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
23:52:10.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.750288 25008 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:10.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.751783 25008 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.751852 25008 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
23:52:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.752179 25030 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap starting.
23:52:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.763751 25008 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.65:39389
23:52:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.763803 25137 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.65:39389 every 8 connection(s)
23:52:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.764791 25008 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.766268 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 25008
23:52:10.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.772423 25030 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Log is configured to *not* fsync() on all Append() calls
23:52:10.775 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:10.775 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.67:35111
23:52:10.776 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.776563 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 24641
23:52:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.776657 25139 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.776872 25139 heartbeater.cc:461] Registering TS with master...
23:52:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.777446 25139 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.777766 25138 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.777845 25138 heartbeater.cc:461] Registering TS with master...
23:52:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.778026 25138 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.779743 23199 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.780650 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.65:50975
23:52:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.780632 23345 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.783288 25140 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.783401 25140 heartbeater.cc:461] Registering TS with master...
23:52:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.783627 25140 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.784472 23271 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:10.789 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] handling channelInactive
23:52:10.789 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state READY due to: connection closed
23:52:10.789 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:10.790 [DEBUG - Time-limited test] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:10.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.792 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.792 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.796590 25030 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:10.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.797088 25030 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap complete.
23:52:10.798 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:10.798 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:10.798 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test)) after lookup
23:52:10.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.798458 25030 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent bootstrapping tablet: real 0.047s	user 0.033s	sys 0.004s
23:52:10.799 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:10.800 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.800 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.801301 25030 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.801685 25030 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Initialized, Role: FOLLOWER
23:52:10.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.802189 25030 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:10.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.802838 25139 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:52:10.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.803053 25030 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Time spent starting tablet: real 0.005s	user 0.007s	sys 0.000s
23:52:10.818 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:10.818 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:10.818 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816))) after lookup
23:52:10.819 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:10.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.821 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:10.858 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:10.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [31ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975))) after lookup
23:52:10.859 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:10.859 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.860 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.860 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.897 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:10.898 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:10.898 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [31ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [68ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [70ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [70ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [107ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540))) after lookup
23:52:10.899 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:10.900 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.900 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:10.900 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:11.378 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:11.379 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=588), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [31ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [68ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [70ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [70ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [107ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [108ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [110ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [110ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [588ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519))) after lookup
23:52:11.379 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:11.380 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.381 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:11.439 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:11.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=649), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [31ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [68ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [70ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [70ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [107ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [108ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [110ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [110ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [588ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [589ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [591ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [591ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [646ms] refreshing cache from master, [646ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [648ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319))) after lookup
23:52:11.440 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:11.441 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.441 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.441 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:11.598 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:11.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=808), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [31ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [68ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [70ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [70ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [107ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [108ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [110ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [110ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [588ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [589ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [591ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [591ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [646ms] refreshing cache from master, [646ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [648ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [650ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [651ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [652ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [805ms] refreshing cache from master, [805ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [808ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564))) after lookup
23:52:11.599 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:11.600 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.600 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:11.600 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111
23:52:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.098943 25151 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 8b14a904198a4e7fab4c5c987d1627dc)
23:52:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.099080 25151 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.100291 25151 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.101354 24844 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:52:12.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.102427 24844 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:52:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.105471 25091 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55" is_pre_election: true
23:52:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.105791 25091 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate acfcc37b94ba4725bc4d48a97a10e613 in term 7.
23:52:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.106601 24844 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 8b14a904198a4e7fab4c5c987d1627dc
23:52:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.106911 25151 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Leader pre-election won for term 9
23:52:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.106999 25151 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 8b14a904198a4e7fab4c5c987d1627dc)
23:52:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.107046 25151 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 8 FOLLOWER]: Advancing to term 9
23:52:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.107940 25151 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.108363 25151 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 9 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.108671 25091 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "acfcc37b94ba4725bc4d48a97a10e613" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:52:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.108803 25091 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 7 FOLLOWER]: Advancing to term 9
23:52:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.109493 24844 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111)
23:52:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.111016 25091 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate acfcc37b94ba4725bc4d48a97a10e613 in term 9.
23:52:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.111351 24844 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, acfcc37b94ba4725bc4d48a97a10e613; no voters: 8b14a904198a4e7fab4c5c987d1627dc
23:52:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.111591 25151 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 FOLLOWER]: Leader election won for term 9
23:52:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.111814 25151 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 LEADER]: Becoming Leader. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Running, Role: LEADER
23:52:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.112112 25151 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:12.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.115219 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 reported cstate change: term changed from 8 to 9, leader changed from 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67) to acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66). New cstate: current_term: 9 leader_uuid: "acfcc37b94ba4725bc4d48a97a10e613" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:52:12.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.613930 24844 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 -> Peer 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111): Couldn't send request to peer 8b14a904198a4e7fab4c5c987d1627dc. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.67:35111: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:12.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.650002 25091 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Refusing update from remote peer acfcc37b94ba4725bc4d48a97a10e613: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
23:52:12.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.650667 25151 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
23:52:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.656594 25173 mvcc.cc:204] Tried to move back new op lower bound from 7305086282119254016 to 7305086272264384512. Current Snapshot: MvccSnapshot[applied={T|T < 7305086282056351744}]
23:52:13.098 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:13.098 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:13.098 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=16, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2308), Traces: [0ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [2ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [8ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [10ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [29ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [31ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [31ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [68ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [70ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [70ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [107ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [108ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [110ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [110ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [588ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [589ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [591ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [591ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [646ms] refreshing cache from master, [646ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [648ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [650ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [651ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [652ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [805ms] refreshing cache from master, [805ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [808ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [809ms] sending RPC to server 8b14a904198a4e7fab4c5c987d1627dc, [810ms] received response from server 8b14a904198a4e7fab4c5c987d1627dc: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [810ms] delaying RPC due to: Network error: Failed to connect to peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111): Connection refused: /127.22.157.67:35111, [2305ms] refreshing cache from master, [2305ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [2307ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1840925447(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@1cfc55e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564) -> (continuation of Deferred@337021542 after retry RPC@1478290429), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66c36136 -> wakeup thread Time-limited test -> (continuation of Deferred@742618708 after retry RPC@1661769816) -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564) -> (continuation of Deferred@337021542 after retry RPC@1478290429))) after lookup
23:52:13.099 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.100 [DEBUG - kudu-nio-7] (Connection.java:757) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] Successfully connected to peer
23:52:13.114 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xc7cbd848, L:/127.0.0.1:51832 - R:/127.22.157.66:37793] using TOKEN/null
23:52:13.131 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.67:35111
23:52:13.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.132179 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:35111
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45069
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.236512 25183 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.236778 25183 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.236826 25183 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.240332 25183 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.240442 25183 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.67
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.244355 25183 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.67:35111
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:52:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.67
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45069
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25183
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.67
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/logs
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.245486 25183 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.246869 25183 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.253764 25183 server_base.cc:1061] running on GCE node
23:52:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.253654 25188 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.253619 25189 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:13.253652 25191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.254304 25183 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.254843 25183 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.255977 25183 hybrid_clock.cc:648] HybridClock initialized: now 1783468333255962 us; error 38 us; skew 500 ppm
23:52:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.257751 25183 webserver.cc:534] Webserver started at http://127.22.157.67:45069/ using document root <none> and password file <none>
23:52:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.258265 25183 fs_manager.cc:362] Metadata directory not provided
23:52:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.258316 25183 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.261607 25183 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.000s
23:52:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.263550 25197 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:13.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.264617 25183 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.264710 25183 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.265082 25183 fs_report.cc:389] FS layout report
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/wal
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/data
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.280072 25183 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.280720 25183 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:13.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.280894 25183 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:13.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.281414 25183 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:13.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.282598 25204 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.286126 25183 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.286195 25183 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
23:52:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.286245 25183 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.287748 25183 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:13.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.287794 25183 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
23:52:13.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.288260 25204 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap starting.
23:52:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.298830 25183 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.67:35111
23:52:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.299095 25312 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.67:35111 every 8 connection(s)
23:52:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.299979 25183 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-2/data/info.pb
23:52:13.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.305966 25204 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Log is configured to *not* fsync() on all Append() calls
23:52:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.308699 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 25183
23:52:13.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.314463 25313 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.314463 25314 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.314754 25314 heartbeater.cc:461] Registering TS with master...
23:52:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.315289 25314 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.314755 25313 heartbeater.cc:461] Registering TS with master...
23:52:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.316926 25313 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:13.317 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:13.318 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.66:37793
23:52:13.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.317358 23199 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.318269 23199 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.67:41841
23:52:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.318890 25315 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.319080 25315 heartbeater.cc:461] Registering TS with master...
23:52:13.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.319799 23344 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:13.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.320124 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 24825
23:52:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.321874 25315 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.322825 23271 ts_manager.cc:194] Re-registered known tserver with Master: 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:13.327 [DEBUG - kudu-nio-7] (Connection.java:239) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] handling channelInactive
23:52:13.327 [DEBUG - kudu-nio-7] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state READY due to: connection closed
23:52:13.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:13.328 [DEBUG - Time-limited test] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.330 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.332006 25204 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:13.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.332441 25204 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Bootstrap complete.
23:52:13.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.333551 25204 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent bootstrapping tablet: real 0.046s	user 0.032s	sys 0.008s
23:52:13.337 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:13.338 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:13.338 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test)) after lookup
23:52:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.335938 25204 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.336396 25204 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Initialized, Role: FOLLOWER
23:52:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.336918 25204 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.337931 25204 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
23:52:13.338 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.339813 25314 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:52:13.342 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.342 [DEBUG - kudu-nio-1] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.342 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.358 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:13.358 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:13.359 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [14ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [14ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778))) after lookup
23:52:13.359 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.360 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.361 [DEBUG - kudu-nio-2] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.361 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:13.378 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:13.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [14ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [14ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [31ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [33ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [33ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021))) after lookup
23:52:13.379 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.380 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.380 [DEBUG - kudu-nio-3] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.380 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.417 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:13.418 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:13.418 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=89), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [14ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [14ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [31ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [33ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [33ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [52ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [52ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [89ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138))) after lookup
23:52:13.419 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.420 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.420 [DEBUG - kudu-nio-4] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.420 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:13.558 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:13.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=230), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [14ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [14ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [31ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [33ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [33ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [52ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [52ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [89ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [90ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [92ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [92ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [227ms] refreshing cache from master, [227ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [229ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223))) after lookup
23:52:13.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:13.560 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.560 [DEBUG - kudu-nio-5] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:13.560 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:14.478 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:14.478 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)[L]]}]
23:52:14.478 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1150), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [14ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [14ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [31ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [33ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [33ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [52ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [52ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [89ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [90ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [92ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [92ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [227ms] refreshing cache from master, [227ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [229ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [231ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [232ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [232ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1147ms] refreshing cache from master, [1147ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [1149ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223) -> (continuation of Deferred@673225744 after retry RPC@674373503), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223) -> (continuation of Deferred@673225744 after retry RPC@674373503))) after lookup
23:52:14.479 [DEBUG - kudu-nio-0] (Connection.java:748) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] connecting to peer
23:52:14.480 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:14.480 [DEBUG - kudu-nio-6] (Connection.java:709) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] cleaning up while in state CONNECTING due to: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:14.480 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793
23:52:14.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.632020 25327 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader acfcc37b94ba4725bc4d48a97a10e613)
23:52:14.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.632223 25327 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:14.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.633379 25327 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.634158 25024 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.635095 25024 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.638240 25266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc" is_pre_election: true
23:52:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.638571 25266 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 8.
23:52:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.639283 25027 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc; no voters: acfcc37b94ba4725bc4d48a97a10e613
23:52:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.639583 25327 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Leader pre-election won for term 10
23:52:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.639639 25327 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Starting leader election (detected failure of leader acfcc37b94ba4725bc4d48a97a10e613)
23:52:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.639668 25327 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 9 FOLLOWER]: Advancing to term 10
23:52:14.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.640482 25327 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.640836 25327 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 10 election: Requested vote from peers acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793), 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67:35111)
23:52:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.641203 25266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "63bc8b354cf246ea9511efd387cf8a55" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "8b14a904198a4e7fab4c5c987d1627dc"
23:52:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.641338 25266 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 8 FOLLOWER]: Advancing to term 10
23:52:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.642004 25024 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111)
23:52:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.642984 25266 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bc8b354cf246ea9511efd387cf8a55 in term 10.
23:52:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.643272 25027 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 63bc8b354cf246ea9511efd387cf8a55, 8b14a904198a4e7fab4c5c987d1627dc; no voters: acfcc37b94ba4725bc4d48a97a10e613
23:52:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.643481 25327 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 10 FOLLOWER]: Leader election won for term 10
23:52:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.643661 25327 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [term 10 LEADER]: Becoming Leader. State: Replica: 63bc8b354cf246ea9511efd387cf8a55, State: Running, Role: LEADER
23:52:14.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.643967 25327 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.646816 23199 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 reported cstate change: term changed from 9 to 10, leader changed from acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66) to 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65). New cstate: current_term: 10 leader_uuid: "63bc8b354cf246ea9511efd387cf8a55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: UNKNOWN } } }
23:52:15.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:15.031961 25024 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 -> Peer acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793): Couldn't send request to peer acfcc37b94ba4725bc4d48a97a10e613. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.66:37793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:15.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:15.044289 25266 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Refusing update from remote peer 63bc8b354cf246ea9511efd387cf8a55: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
23:52:15.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:15.045595 25327 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
23:52:17.357 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:17.358 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:17.358 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=4029), Traces: [0ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [2ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [2ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [9ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [10ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [14ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [14ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [31ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [33ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [33ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [50ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [52ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [52ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [89ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [90ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [92ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [92ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [227ms] refreshing cache from master, [227ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [229ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [231ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [232ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [232ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1147ms] refreshing cache from master, [1147ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [1149ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [1151ms] sending RPC to server acfcc37b94ba4725bc4d48a97a10e613, [1152ms] received response from server acfcc37b94ba4725bc4d48a97a10e613: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [1152ms] delaying RPC due to: Network error: Failed to connect to peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793): Connection refused: /127.22.157.66:37793, [4026ms] refreshing cache from master, [4027ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [4029ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223) -> (continuation of Deferred@673225744 after retry RPC@674373503) -> (continuation of Deferred@1992554065 after retry RPC@1834755163), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223) -> (continuation of Deferred@673225744 after retry RPC@674373503) -> (continuation of Deferred@1992554065 after retry RPC@1834755163))) after lookup
23:52:17.359 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:17.360 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] Successfully connected to peer
23:52:17.374 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x1003ad37, L:/127.0.0.1:41072 - R:/127.22.157.65:39389] using TOKEN/null
23:52:17.391 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.66:37793
23:52:17.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.392833 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:37793
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43755
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.497195 25370 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.497421 25370 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.497480 25370 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.500973 25370 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.501114 25370 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.66
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.505110 25370 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.66:37793
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.66
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43755
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25370
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.66
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/logs
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.506521 25370 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.507905 25370 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.514315 25377 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.514351 25380 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:17.514339 25378 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.514593 25370 server_base.cc:1061] running on GCE node
23:52:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.515470 25370 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.516002 25370 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.517164 25370 hybrid_clock.cc:648] HybridClock initialized: now 1783468337517147 us; error 35 us; skew 500 ppm
23:52:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.518922 25370 webserver.cc:534] Webserver started at http://127.22.157.66:43755/ using document root <none> and password file <none>
23:52:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.519496 25370 fs_manager.cc:362] Metadata directory not provided
23:52:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.519555 25370 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.522807 25370 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
23:52:17.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.524682 25386 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.525626 25370 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.525717 25370 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.526085 25370 fs_report.cc:389] FS layout report
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/wal
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/data
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:17.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.538645 25370 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.539324 25370 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.539553 25370 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.540092 25370 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.541277 25393 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.544665 25370 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.544766 25370 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
23:52:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.544821 25370 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:17.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.546267 25370 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:17.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.546314 25370 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
23:52:17.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.546677 25393 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap starting.
23:52:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.557716 25370 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.66:37793
23:52:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.557754 25501 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.66:37793 every 8 connection(s)
23:52:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.558751 25370 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-1/data/info.pb
23:52:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.559567 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 25370
23:52:17.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.564747 25393 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Log is configured to *not* fsync() on all Append() calls
23:52:17.569 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:17.570 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.157.65:39389
23:52:17.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.571084 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 25008
23:52:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.574107 25503 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.574381 25503 heartbeater.cc:461] Registering TS with master...
23:52:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.574772 25502 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.574870 25502 heartbeater.cc:461] Registering TS with master...
23:52:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.575023 25503 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.575090 25502 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:17.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.577111 23345 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:17.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.577132 23197 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.578109 23197 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.66:44297
23:52:17.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.581012 25504 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:17.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.581112 25504 heartbeater.cc:461] Registering TS with master...
23:52:17.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.581287 25504 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:17.582 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] handling channelInactive
23:52:17.582 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state READY due to: connection closed
23:52:17.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.582209 23272 ts_manager.cc:194] Re-registered known tserver with Master: acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:17.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:17.583 [DEBUG - Time-limited test] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:17.584 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.585 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.595037 25393 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.595580 25393 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Bootstrap complete.
23:52:17.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.596932 25393 ts_tablet_manager.cc:1403] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent bootstrapping tablet: real 0.051s	user 0.030s	sys 0.012s
23:52:17.597 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:17.598 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:17.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [14ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1210505249(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@550e44f5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61e6647b -> wakeup thread Time-limited test)) after lookup
23:52:17.599 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.599813 25393 raft_consensus.cc:359] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.600186 25393 raft_consensus.cc:740] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: acfcc37b94ba4725bc4d48a97a10e613, State: Initialized, Role: FOLLOWER
23:52:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.600791 25393 consensus_queue.cc:260] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:17.601 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.601 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.601 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.601740 25393 ts_tablet_manager.cc:1434] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
23:52:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:17.602188 25503 heartbeater.cc:499] Master 127.22.157.126:38363 was elected leader, sending a full tablet report...
23:52:17.618 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:17.618 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:17.618 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [14ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [15ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [18ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [18ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [34ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1210505249(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@550e44f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61e6647b -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992))) after lookup
23:52:17.619 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:17.620 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.620 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.620 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:17.658 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:17.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [14ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [15ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [18ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [18ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [34ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [35ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [37ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [37ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [74ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1210505249(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@550e44f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61e6647b -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650))) after lookup
23:52:17.659 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:17.660 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.660 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.660 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.778 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:17.779 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:17.779 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=196), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [14ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [15ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [18ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [18ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [34ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [35ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [37ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [37ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [74ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [75ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [77ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [77ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [195ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1210505249(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@550e44f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61e6647b -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024))) after lookup
23:52:17.780 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:17.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:17.781 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:18.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:18.278 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)[L],8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111),acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:18.279 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=695), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [14ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [15ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [18ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [18ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [34ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [35ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [37ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [37ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [74ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [75ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [77ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [77ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [195ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [196ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [198ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [198ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [692ms] refreshing cache from master, [692ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [695ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1210505249(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@550e44f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024) -> (continuation of Deferred@1568412073 after retry RPC@1338189423), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61e6647b -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024) -> (continuation of Deferred@1568412073 after retry RPC@1338189423))) after lookup
23:52:18.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] connecting to peer
23:52:18.281 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:18.281 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] cleaning up while in state CONNECTING due to: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:18.281 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389) for tablet 7ece0c8f846e439aaa0a323ed2d0c121: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389
23:52:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.891660 25515 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:52:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.891876 25515 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:18.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.893028 25515 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:18.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.893916 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.894835 25201 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.898290 25454 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613" is_pre_election: true
23:52:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.898638 25454 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 9.
23:52:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.899471 25198 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:52:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.899730 25515 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Leader pre-election won for term 11
23:52:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.899789 25515 raft_consensus.cc:493] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Starting leader election (detected failure of leader 63bc8b354cf246ea9511efd387cf8a55)
23:52:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.899820 25515 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 10 FOLLOWER]: Advancing to term 11
23:52:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.900594 25515 raft_consensus.cc:515] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.900986 25515 leader_election.cc:290] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 11 election: Requested vote from peers 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389), acfcc37b94ba4725bc4d48a97a10e613 (127.22.157.66:37793)
23:52:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.901599 25454 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ece0c8f846e439aaa0a323ed2d0c121" candidate_uuid: "8b14a904198a4e7fab4c5c987d1627dc" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "acfcc37b94ba4725bc4d48a97a10e613"
23:52:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.901737 25454 raft_consensus.cc:3060] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 9 FOLLOWER]: Advancing to term 11
23:52:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.902140 25201 leader_election.cc:336] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111)
23:52:18.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.903674 25454 raft_consensus.cc:2468] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b14a904198a4e7fab4c5c987d1627dc in term 11.
23:52:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.904039 25198 leader_election.cc:304] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b14a904198a4e7fab4c5c987d1627dc, acfcc37b94ba4725bc4d48a97a10e613; no voters: 63bc8b354cf246ea9511efd387cf8a55
23:52:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.904230 25515 raft_consensus.cc:2804] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 11 FOLLOWER]: Leader election won for term 11
23:52:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.904475 25515 raft_consensus.cc:697] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [term 11 LEADER]: Becoming Leader. State: Replica: 8b14a904198a4e7fab4c5c987d1627dc, State: Running, Role: LEADER
23:52:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.904798 25515 consensus_queue.cc:237] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } }
23:52:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.907297 23197 catalog_manager.cc:5696] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc reported cstate change: term changed from 10 to 11, leader changed from 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65) to 8b14a904198a4e7fab4c5c987d1627dc (127.22.157.67). New cstate: current_term: 11 leader_uuid: "8b14a904198a4e7fab4c5c987d1627dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bc8b354cf246ea9511efd387cf8a55" member_type: VOTER last_known_addr { host: "127.22.157.65" port: 39389 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8b14a904198a4e7fab4c5c987d1627dc" member_type: VOTER last_known_addr { host: "127.22.157.67" port: 35111 } health_report { overall_health: HEALTHY } } }
23:52:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.305766 25454 raft_consensus.cc:1275] T 7ece0c8f846e439aaa0a323ed2d0c121 P acfcc37b94ba4725bc4d48a97a10e613 [term 11 FOLLOWER]: Refusing update from remote peer 8b14a904198a4e7fab4c5c987d1627dc: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
23:52:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.306414 25515 consensus_queue.cc:1048] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "acfcc37b94ba4725bc4d48a97a10e613" member_type: VOTER last_known_addr { host: "127.22.157.66" port: 37793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
23:52:19.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.314424 25201 consensus_peers.cc:597] T 7ece0c8f846e439aaa0a323ed2d0c121 P 8b14a904198a4e7fab4c5c987d1627dc -> Peer 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389): Couldn't send request to peer 63bc8b354cf246ea9511efd387cf8a55. Status: Network error: Client connection negotiation failed: client connection to 127.22.157.65:39389: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:19.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7ece0c8f846e439aaa0a323ed2d0c121 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982' with partition [<start>, <end>)
23:52:19.879 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ece0c8f846e439aaa0a323ed2d0c121@[63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389),8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)[L],acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)]}]
23:52:19.879 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ece0c8f846e439aaa0a323ed2d0c121, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2296), Traces: [0ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [2ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [14ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [15ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [18ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [18ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [34ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [35ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [37ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [37ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [74ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [75ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [77ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [77ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [195ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [196ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [198ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [198ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [692ms] refreshing cache from master, [692ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [695ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, [696ms] sending RPC to server 63bc8b354cf246ea9511efd387cf8a55, [698ms] received response from server 63bc8b354cf246ea9511efd387cf8a55: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [698ms] delaying RPC due to: Network error: Failed to connect to peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389): Connection refused: /127.22.157.65:39389, [2292ms] refreshing cache from master, [2292ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.157.126:38363, [2295ms] Sub RPC GetTableLocations: received response from server master-127.22.157.126:38363: OK, deferred=Deferred@1210505249(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007fbe9836bed0@550e44f5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024) -> (continuation of Deferred@1568412073 after retry RPC@1338189423) -> (continuation of Deferred@921885671 after retry RPC@1616147208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61e6647b -> wakeup thread Time-limited test -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024) -> (continuation of Deferred@1568412073 after retry RPC@1338189423) -> (continuation of Deferred@921885671 after retry RPC@1616147208))) after lookup
23:52:19.880 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] connecting to peer
23:52:19.881 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] Successfully connected to peer
23:52:19.896 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x6d25d010, L:/127.0.0.1:35050 - R:/127.22.157.67:35111] using TOKEN/null
23:52:19.913 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.157.65:39389
23:52:19.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:19.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.913745 23157 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:52:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:19.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:19.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:19.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:20.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.016625 25549 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:20.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.016847 25549 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:20.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.016892 25549 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:20.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.020274 25549 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:20.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.020380 25549 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.157.65
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.024115 25549 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.157.84:40527
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.157.65:39389
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.157.65
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.157.126:38363,127.22.157.125:42957,127.22.157.124:43683
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25549
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.157.65
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/logs
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
23:52:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:33 UTC on bdcb31816ec0
23:52:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16833
23:52:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.025203 25549 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.026295 25549 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.032754 25557 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.032760 25554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.032773 25555 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.032824 25549 server_base.cc:1061] running on GCE node
23:52:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.033659 25549 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.034158 25549 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:20.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.035306 25549 hybrid_clock.cc:648] HybridClock initialized: now 1783468340035287 us; error 40 us; skew 500 ppm
23:52:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.037096 25549 webserver.cc:534] Webserver started at http://127.22.157.65:40325/ using document root <none> and password file <none>
23:52:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.037588 25549 fs_manager.cc:362] Metadata directory not provided
23:52:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.037639 25549 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.040824 25549 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
23:52:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.042548 25563 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.043705 25549 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.043805 25549 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data,/tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bc8b354cf246ea9511efd387cf8a55"
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:49 on dist-test-slave-3q43"
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.044183 25549 fs_report.cc:389] FS layout report
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/wal
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/data
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.077387 25549 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:20.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.078109 25549 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:20.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.078274 25549 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:20.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.078805 25549 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:20.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.080102 25570 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:52:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.083545 25549 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.083616 25549 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
23:52:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.083673 25549 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.085155 25549 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.085201 25549 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
23:52:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.085569 25570 tablet_bootstrap.cc:492] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Bootstrap starting.
23:52:20.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.096526 25549 rpc_server.cc:307] RPC server started. Bound to: 127.22.157.65:39389
23:52:20.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.096544 25679 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.157.65:39389 every 8 connection(s)
23:52:20.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.097544 25549 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskUpAgIg/test-tmp/mini-kudu-cluster9003924256090654436/ts-0/data/info.pb
23:52:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.100562 23157 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu as pid 25549
23:52:20.101 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.103763 25570 log.cc:826] T 7ece0c8f846e439aaa0a323ed2d0c121 P 63bc8b354cf246ea9511efd387cf8a55: Log is configured to *not* fsync() on all Append() calls
23:52:20.109 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.112459 25680 heartbeater.cc:344] Connected to a master server at 127.22.157.124:43683
23:52:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.112710 25680 heartbeater.cc:461] Registering TS with master...
23:52:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.113304 25680 heartbeater.cc:507] Master 127.22.157.124:43683 requested a full tablet report, sending...
23:52:20.114 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468309982, tablet=7ece0c8f846e439aaa0a323ed2d0c121, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:20.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.114596 25682 heartbeater.cc:344] Connected to a master server at 127.22.157.125:42957
23:52:20.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.114691 25682 heartbeater.cc:461] Registering TS with master...
23:52:20.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.114925 25682 heartbeater.cc:507] Master 127.22.157.125:42957 requested a full tablet report, sending...
23:52:20.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.116082 23272 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:20.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.117053 25681 heartbeater.cc:344] Connected to a master server at 127.22.157.126:38363
23:52:20.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.117172 25681 heartbeater.cc:461] Registering TS with master...
23:52:20.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.117376 25681 heartbeater.cc:507] Master 127.22.157.126:38363 requested a full tablet report, sending...
23:52:20.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.118180 23197 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:20.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.118984 23197 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.157.65:52093
23:52:20.119 [DEBUG - main] (Connection.java:575) [peer acfcc37b94ba4725bc4d48a97a10e613(127.22.157.66:37793)] disconnecting while in state TERMINATED
23:52:20.120 [DEBUG - main] (Connection.java:575) [peer master-127.22.157.124:43683(127.22.157.124:43683)] disconnecting while in state READY
23:52:20.121 [DEBUG - main] (Connection.java:575) [peer 63bc8b354cf246ea9511efd387cf8a55(127.22.157.65:39389)] disconnecting while in state TERMINATED
23:52:20.121 [DEBUG - main] (Connection.java:575) [peer master-127.22.157.125:42957(127.22.157.125:42957)] disconnecting while in state READY
23:52:20.121 [DEBUG - main] (Connection.java:575) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] disconnecting while in state READY
23:52:20.121 [DEBUG - main] (Connection.java:575) [peer master-127.22.157.126:38363(127.22.157.126:38363)] disconnecting while in state READY
23:52:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.122561 23345 ts_manager.cc:194] Re-registered known tserver with Master: 63bc8b354cf246ea9511efd387cf8a55 (127.22.157.65:39389)
23:52:20.124 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] handling channelInactive
23:52:20.124 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8b14a904198a4e7fab4c5c987d1627dc(127.22.157.67:35111)] cleaning up while in state READY due to: connection closed
23:52:20.124 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.157.125:42957(127.22.157.125:42957)] handling channelInactive
23:52:20.124 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.157.125:42957(127.22.157.125:42957)] cleaning up while in state READY due to: connection closed
23:52:20.125 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.157.126:38363(127.22.157.126:38363)] handling channelInactive
23:52:20.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.157.126:38363(127.22.157.126:38363)] cleaning up while in state READY due to: connection closed
23:52:20.125 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:52:20.127 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.22.157.124:43683(127.22.157.124:43683)] handling channelInactive
23:52:20.127 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.22.157.124:43683(127.22.157.124:43683)] cleaning up while in state READY due to: connection closed
23:52:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.128854 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 25549
23:52:20.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.138181 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 25370
23:52:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.146643 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 25183
23:52:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.153057 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23168
23:52:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.159521 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23237
23:52:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:20.165541 23157 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskUpAgIg/build/debug/bin/kudu with pid 23312
23:52:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:52:20Z chronyd exiting

Time: 32.938

OK (1 test)

