JUnit version 4.13.2
.23:00:37.257 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783465237247
23:00:37.331 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
23:00:37.362 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:00:37.373 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskjzpEJp/test-tmp
23:00:37.397 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
23:00:37.399 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskjzpEJp/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
23:00:37.626 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

23:00:37.817 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:00:37.819 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

23:00:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:37.822109 12001 mini_oidc.cc:150] Starting JWKS server
23:00:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:37.824721 12001 webserver.cc:534] Webserver started at http://127.0.0.1:42171/ using document root <none> and password file <none>
23:00:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:37.825037 12001 mini_oidc.cc:186] Starting OIDC Discovery server
23:00:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:37.827445 12001 webserver.cc:534] Webserver started at http://127.0.0.1:39361/ using document root <none> and password file <none>
23:00:37.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:00:37Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
23:00:37.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:00:37Z Disabled control of system clock
23:00:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:37.857790 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal
23:00:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data
23:00:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:37.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:37.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:37.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/logs
23:00:37.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data/info.pb
23:00:37.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:37.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:37.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:00:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:00:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:00:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.126:46381
23:00:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.126
23:00:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:00:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:00:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42171/jwks.json with env {}
23:00:38.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.204576 12012 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:38.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.204981 12012 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.205099 12012 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.214174 12012 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:00:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.214359 12012 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.214468 12012 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:00:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.214538 12012 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:00:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.214624 12012 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:00:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.214679 12012 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42171/jwks.json
23:00:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.226540 12012 master_runner.cc:387] Master server non-default flags:
23:00:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data
23:00:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal
23:00:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:00:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:00:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.126:46381
23:00:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:00:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:00:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42171/jwks.json
23:00:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:38.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data/info.pb
23:00:38.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.126
23:00:38.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:38.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:38.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/logs
23:00:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:00:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.228932 12012 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:38.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.231526 12012 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:38.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.241842 12017 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:38.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.241786 12018 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:38.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.241750 12020 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:38.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.243197 12012 server_base.cc:1061] running on GCE node
23:00:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.244211 12012 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.245776 12012 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.247012 12012 hybrid_clock.cc:648] HybridClock initialized: now 1783465238246951 us; error 53 us; skew 500 ppm
23:00:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.247483 12012 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:38.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.249905 12012 webserver.cc:534] Webserver started at http://127.11.184.126:42375/ using document root <none> and password file <none>
23:00:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.250859 12012 fs_manager.cc:362] Metadata directory not provided
23:00:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.250994 12012 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.251619 12012 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:00:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.254120 12012 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data/instance:
23:00:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20c53519cb0d49d6beda8fb455524abf"
23:00:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:38 on dist-test-slave-1pcn"
23:00:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.255072 12012 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal/instance:
23:00:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20c53519cb0d49d6beda8fb455524abf"
23:00:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:38 on dist-test-slave-1pcn"
23:00:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.260677 12012 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.004s	sys 0.000s
23:00:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.264413 12026 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.265954 12012 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
23:00:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.266188 12012 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal
23:00:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20c53519cb0d49d6beda8fb455524abf"
23:00:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:38 on dist-test-slave-1pcn"
23:00:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.266444 12012 fs_report.cc:389] FS layout report
23:00:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal
23:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal
23:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data/data
23:00:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:38.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.290701 12012 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:38.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.291944 12012 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:38.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.292306 12012 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.314680 12012 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.126:46381
23:00:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.314718 12077 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.126:46381 every 8 connection(s)
23:00:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.316682 12012 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/data/info.pb
23:00:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.319893 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12012
23:00:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.320237 12001 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-0/wal/instance
23:00:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.322273 12078 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:00:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.323284 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/logs
23:00:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data/info.pb
23:00:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:00:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:00:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:00:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.125:41661
23:00:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.125
23:00:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:00:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:00:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42171/jwks.json with env {}
23:00:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.330212 12078 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } has no permanent_uuid. Determining permanent_uuid...
23:00:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.353259 12078 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } has no permanent_uuid. Determining permanent_uuid...
23:00:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.354525 12030 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.125:41661: connect: Connection refused (error 111)
23:00:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.356402 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.125:41661: Network error: Client connection negotiation failed: client connection to 127.11.184.125:41661: connect: Connection refused (error 111)
23:00:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.405854 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } attempt: 1
23:00:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.408595 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.125:41661: Network error: Client connection negotiation failed: client connection to 127.11.184.125:41661: connect: Connection refused (error 111)
23:00:38.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.477006 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } attempt: 2
23:00:38.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.479460 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.125:41661: Network error: Client connection negotiation failed: client connection to 127.11.184.125:41661: connect: Connection refused (error 111)
23:00:38.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.570835 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } attempt: 3
23:00:38.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.573431 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.125:41661: Network error: Client connection negotiation failed: client connection to 127.11.184.125:41661: connect: Connection refused (error 111)
23:00:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.659619 12080 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.659969 12080 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.660087 12080 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:38.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.668537 12080 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:00:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.668669 12080 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.668751 12080 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:00:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.668808 12080 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:00:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.668880 12080 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:00:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.668929 12080 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42171/jwks.json
23:00:38.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.679791 12080 master_runner.cc:387] Master server non-default flags:
23:00:38.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:38.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.125:41661
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:00:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42171/jwks.json
23:00:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data/info.pb
23:00:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.125
23:00:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/logs
23:00:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:00:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.682116 12080 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.684695 12080 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.693861 12088 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.694397 12089 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.696728 12091 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.697181 12080 server_base.cc:1061] running on GCE node
23:00:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.698064 12080 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.699798 12080 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.701049 12080 hybrid_clock.cc:648] HybridClock initialized: now 1783465238700937 us; error 91 us; skew 500 ppm
23:00:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.701489 12080 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.703907 12080 webserver.cc:534] Webserver started at http://127.11.184.125:45243/ using document root <none> and password file <none>
23:00:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.704770 12080 fs_manager.cc:362] Metadata directory not provided
23:00:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.704903 12080 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.705375 12080 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:00:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.707923 12080 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data/instance:
23:00:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149"
23:00:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:38 on dist-test-slave-1pcn"
23:00:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.708879 12080 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal/instance:
23:00:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149"
23:00:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:38 on dist-test-slave-1pcn"
23:00:38.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.716655 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } attempt: 4
23:00:38.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.718957 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.125:41661: Network error: Client connection negotiation failed: client connection to 127.11.184.125:41661: connect: Connection refused (error 111)
23:00:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.720541 12080 fs_manager.cc:696] Time spent creating directory manager: real 0.011s	user 0.006s	sys 0.004s
23:00:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.724288 12097 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.725857 12080 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
23:00:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.726061 12080 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal
23:00:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149"
23:00:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:38 on dist-test-slave-1pcn"
23:00:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.726322 12080 fs_report.cc:389] FS layout report
23:00:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal
23:00:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal
23:00:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data/data
23:00:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.743858 12080 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.744885 12080 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.745334 12080 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:38.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.766494 12080 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.125:41661
23:00:38.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.766512 12148 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.125:41661 every 8 connection(s)
23:00:38.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.768353 12080 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/data/info.pb
23:00:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.768813 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12080
23:00:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.768944 12001 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-1/wal/instance
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.770277 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/logs
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data/info.pb
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:00:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:00:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:00:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.124:42425
23:00:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.124
23:00:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:00:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:00:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42171/jwks.json with env {}
23:00:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.775162 12149 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.784667 12149 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } has no permanent_uuid. Determining permanent_uuid...
23:00:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.800617 12149 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } has no permanent_uuid. Determining permanent_uuid...
23:00:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.810241 12149 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } has no permanent_uuid. Determining permanent_uuid...
23:00:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.811733 12099 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.813457 12149 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.862773 12149 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 1
23:00:38.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.865206 12149 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:38.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:38.933450 12149 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 2
23:00:38.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:38.935768 12149 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.018219 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } attempt: 5
23:00:39.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.025007 12078 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } has no permanent_uuid. Determining permanent_uuid...
23:00:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.026938 12149 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 3
23:00:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.027392 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.028730 12149 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.078624 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 1
23:00:39.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.080909 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.117652 12151 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.117971 12151 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.118050 12151 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.126247 12151 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:00:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.126346 12151 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.126402 12151 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:00:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.126441 12151 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:00:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.126493 12151 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:00:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.126535 12151 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42171/jwks.json
23:00:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.137157 12151 master_runner.cc:387] Master server non-default flags:
23:00:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.124:42425
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:00:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42171/jwks.json
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data/info.pb
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.124
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/logs
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.138962 12151 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.141571 12151 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:39.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.149150 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 2
23:00:39.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.150877 12159 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:39.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.151726 12078 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.152949 12160 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.154821 12162 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.155346 12151 server_base.cc:1061] running on GCE node
23:00:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.156164 12151 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.157609 12151 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.158855 12151 hybrid_clock.cc:648] HybridClock initialized: now 1783465239158795 us; error 43 us; skew 500 ppm
23:00:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.159307 12151 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.161667 12151 webserver.cc:534] Webserver started at http://127.11.184.124:35029/ using document root <none> and password file <none>
23:00:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.162477 12151 fs_manager.cc:362] Metadata directory not provided
23:00:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.162606 12151 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.163059 12151 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:00:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.165555 12151 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data/instance:
23:00:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247"
23:00:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:39 on dist-test-slave-1pcn"
23:00:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.166517 12151 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal/instance:
23:00:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247"
23:00:39.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:39 on dist-test-slave-1pcn"
23:00:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.171943 12149 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 4
23:00:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.172241 12151 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.008s	sys 0.000s
23:00:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.174172 12149 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.184.124:42425: Network error: Client connection negotiation failed: client connection to 127.11.184.124:42425: connect: Connection refused (error 111)
23:00:39.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.177018 12168 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.178838 12151 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.000s
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.179025 12151 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247"
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:39 on dist-test-slave-1pcn"
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.179363 12151 fs_report.cc:389] FS layout report
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal
23:00:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data/data
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.201635 12151 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.202483 12151 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.202718 12151 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.223973 12151 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.124:42425
23:00:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.224011 12219 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.124:42425 every 8 connection(s)
23:00:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.225543 12151 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/data/info.pb
23:00:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.229990 12220 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:00:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.230405 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12151
23:00:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.230566 12001 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/master-2/wal/instance
23:00:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.236287 12220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } has no permanent_uuid. Determining permanent_uuid...
23:00:39.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.251588 12220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } has no permanent_uuid. Determining permanent_uuid...
23:00:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.258234 12078 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 3
23:00:39.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.259373 12220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } has no permanent_uuid. Determining permanent_uuid...
23:00:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.270581 12078 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: Bootstrap starting.
23:00:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.274730 12078 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: Neither blocks nor log segments found. Creating new log.
23:00:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.276160 12078 log.cc:826] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: Log is configured to *not* fsync() on all Append() calls
23:00:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.277019 12220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: Bootstrap starting.
23:00:39.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.279434 12078 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: No bootstrap required, opened a new log
23:00:39.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.280849 12220 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: Neither blocks nor log segments found. Creating new log.
23:00:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.282130 12220 log.cc:826] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: Log is configured to *not* fsync() on all Append() calls
23:00:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.286911 12220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: No bootstrap required, opened a new log
23:00:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.287248 12078 raft_consensus.cc:359] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.287583 12078 raft_consensus.cc:385] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:00:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.287734 12078 raft_consensus.cc:740] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20c53519cb0d49d6beda8fb455524abf, State: Initialized, Role: FOLLOWER
23:00:39.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.288427 12078 consensus_queue.cc:260] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.289575 12231 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.290189 12231 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.290875 12078 sys_catalog.cc:565] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: configured and running, proceeding with master startup.
23:00:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.294937 12220 raft_consensus.cc:359] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.295360 12032 tablet.cc:2406] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:00:39.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.295425 12220 raft_consensus.cc:385] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:00:39.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.295599 12220 raft_consensus.cc:740] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9c3bcb8abd843f3b731bb7fa3a5f247, State: Initialized, Role: FOLLOWER
23:00:39.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.296465 12220 consensus_queue.cc:260] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.297760 12240 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.298069 12240 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.299321 12220 sys_catalog.cc:565] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: configured and running, proceeding with master startup.
23:00:39.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.306512 12243 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:00:39.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.307215 12243 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:00:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.305572 12012 master_runner.cc:187] Error getting master registration for 127.11.184.125:41661: OK, instance_id { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" instance_seqno: 1783465238760537 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:00:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.316567 12012 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:00:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.337208 12255 catalog_manager.cc:1593] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:00:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.337970 12255 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:00:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.329829 12151 master_runner.cc:187] Error getting master registration for 127.11.184.125:41661: OK, instance_id { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" instance_seqno: 1783465238760537 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:00:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.340368 12151 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:00:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.440743 12240 raft_consensus.cc:493] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:00:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.441079 12240 raft_consensus.cc:515] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.443063 12240 leader_election.cc:290] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 20c53519cb0d49d6beda8fb455524abf (127.11.184.126:46381), 5a082ae4c3ef42d59cdc4f79ff2c0149 (127.11.184.125:41661)
23:00:39.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.443789 12053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20c53519cb0d49d6beda8fb455524abf" is_pre_election: true
23:00:39.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.444355 12053 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a9c3bcb8abd843f3b731bb7fa3a5f247 in term 0.
23:00:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.445022 12172 leader_election.cc:304] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20c53519cb0d49d6beda8fb455524abf, a9c3bcb8abd843f3b731bb7fa3a5f247; no voters: 
23:00:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.445459 12240 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:00:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.445617 12240 raft_consensus.cc:493] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:00:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.445724 12240 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 0 FOLLOWER]: Advancing to term 1
23:00:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.449409 12240 raft_consensus.cc:515] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.450193 12240 leader_election.cc:290] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [CANDIDATE]: Term 1 election: Requested vote from peers 20c53519cb0d49d6beda8fb455524abf (127.11.184.126:46381), 5a082ae4c3ef42d59cdc4f79ff2c0149 (127.11.184.125:41661)
23:00:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.450716 12053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20c53519cb0d49d6beda8fb455524abf"
23:00:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.450937 12053 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 0 FOLLOWER]: Advancing to term 1
23:00:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:39.454830 12174 tablet.cc:2406] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:00:39.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.456332 12053 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a9c3bcb8abd843f3b731bb7fa3a5f247 in term 1.
23:00:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.457198 12172 leader_election.cc:304] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20c53519cb0d49d6beda8fb455524abf, a9c3bcb8abd843f3b731bb7fa3a5f247; no voters: 
23:00:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.457912 12240 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 1 FOLLOWER]: Leader election won for term 1
23:00:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.461761 12240 raft_consensus.cc:697] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [term 1 LEADER]: Becoming Leader. State: Replica: a9c3bcb8abd843f3b731bb7fa3a5f247, State: Running, Role: LEADER
23:00:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.462827 12240 consensus_queue.cc:237] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.473402 12149 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } attempt: 5
23:00:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.478322 12257 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a9c3bcb8abd843f3b731bb7fa3a5f247. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.479811 12257 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: This master's current role is: LEADER
23:00:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.489624 12149 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: Bootstrap starting.
23:00:39.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.492302 12259 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
23:00:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.495929 12149 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: Neither blocks nor log segments found. Creating new log.
23:00:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.497509 12149 log.cc:826] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: Log is configured to *not* fsync() on all Append() calls
23:00:39.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.505399 12259 catalog_manager.cc:1519] Initializing Kudu cluster ID...
23:00:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.507187 12149 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: No bootstrap required, opened a new log
23:00:39.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.515885 12149 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.517179 12149 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:00:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.517643 12149 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a082ae4c3ef42d59cdc4f79ff2c0149, State: Initialized, Role: FOLLOWER
23:00:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.519294 12149 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } }
23:00:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.528910 12053 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [term 1 FOLLOWER]: Refusing update from remote peer a9c3bcb8abd843f3b731bb7fa3a5f247: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:00:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.530579 12257 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [LEADER]: Connected to new peer: Peer: permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:00:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.537384 12262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.538161 12262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.538877 12149 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: configured and running, proceeding with master startup.
23:00:39.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.538910 12124 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" is_pre_election: true
23:00:39.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.539651 12122 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [term 0 FOLLOWER]: Advancing to term 1
23:00:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.543546 12123 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149"
23:00:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.545974 12122 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [term 1 FOLLOWER]: Refusing update from remote peer a9c3bcb8abd843f3b731bb7fa3a5f247: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:00:39.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.550613 12257 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:00:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.586431 12231 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: SysCatalogTable state changed. Reason: New leader a9c3bcb8abd843f3b731bb7fa3a5f247. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.586752 12231 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.592929 12261 mvcc.cc:204] Tried to move back new op lower bound from 7305073621051981824 to 7305073620847382528. Current Snapshot: MvccSnapshot[applied={T|T < 7305073621051981824}]
23:00:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.593215 12265 mvcc.cc:204] Tried to move back new op lower bound from 7305073621051981824 to 7305073620847382528. Current Snapshot: MvccSnapshot[applied={T|T < 7305073621051981824}]
23:00:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.594673 12262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a9c3bcb8abd843f3b731bb7fa3a5f247. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.594934 12262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.596174 12231 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.596387 12231 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.601243 12264 mvcc.cc:204] Tried to move back new op lower bound from 7305073621051981824 to 7305073620847382528. Current Snapshot: MvccSnapshot[applied={T|T < 7305073621051981824}]
23:00:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.603531 12262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.603775 12262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149 [sys.catalog]: This master's current role is: FOLLOWER
23:00:39.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.614197 12240 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.614522 12240 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: This master's current role is: LEADER
23:00:39.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.614794 12257 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20c53519cb0d49d6beda8fb455524abf" member_type: VOTER last_known_addr { host: "127.11.184.126" port: 46381 } } peers { permanent_uuid: "5a082ae4c3ef42d59cdc4f79ff2c0149" member_type: VOTER last_known_addr { host: "127.11.184.125" port: 41661 } } peers { permanent_uuid: "a9c3bcb8abd843f3b731bb7fa3a5f247" member_type: VOTER last_known_addr { host: "127.11.184.124" port: 42425 } } }
23:00:39.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.614979 12257 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247 [sys.catalog]: This master's current role is: LEADER
23:00:39.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.615942 12259 catalog_manager.cc:1382] Generated new cluster ID: 6db7e06302b248e39ab656325a6b8e51
23:00:39.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.616053 12259 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
23:00:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.636780 12259 catalog_manager.cc:1405] Generated new certificate authority record
23:00:39.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.638733 12259 catalog_manager.cc:1539] Loading token signing keys...
23:00:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.662585 12259 catalog_manager.cc:6069] T 00000000000000000000000000000000 P a9c3bcb8abd843f3b731bb7fa3a5f247: Generated new TSK 0
23:00:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.665174 12259 catalog_manager.cc:1549] Initializing in-progress tserver states...
23:00:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.686869 12281 catalog_manager.cc:1294] Loaded cluster ID: 6db7e06302b248e39ab656325a6b8e51
23:00:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.687497 12281 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: loading cluster ID for follower catalog manager: success
23:00:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.692193 12281 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: acquiring CA information for follower catalog manager: success
23:00:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.695295 12281 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 5a082ae4c3ef42d59cdc4f79ff2c0149: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:00:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:39.701148 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:00:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:0
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:39.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:39.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:39.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:39.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:00:40.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.256870 12283 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:40.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.257197 12283 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:40.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.257285 12283 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:40.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.266917 12283 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:40.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.267102 12283 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.65
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.278813 12283 tablet_server_runner.cc:78] Tablet server non-default flags:
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:00:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:0
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:40.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.281178 12283 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:40.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.283980 12283 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.297220 12288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.297178 12289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.309401 12291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.310544 12283 server_base.cc:1061] running on GCE node
23:00:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.311250 12243 catalog_manager.cc:1294] Loaded cluster ID: 6db7e06302b248e39ab656325a6b8e51
23:00:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.311347 12243 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: loading cluster ID for follower catalog manager: success
23:00:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.311827 12283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.316365 12243 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: acquiring CA information for follower catalog manager: success
23:00:40.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.319430 12243 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 20c53519cb0d49d6beda8fb455524abf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:00:40.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.320286 12283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.321915 12283 hybrid_clock.cc:648] HybridClock initialized: now 1783465240321808 us; error 115 us; skew 500 ppm
23:00:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.322618 12283 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:40.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.330286 12283 webserver.cc:534] Webserver started at http://127.11.184.65:40833/ using document root <none> and password file <none>
23:00:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.331400 12283 fs_manager.cc:362] Metadata directory not provided
23:00:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.331539 12283 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.331944 12283 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:00:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.335798 12283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/instance:
23:00:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82b47e11f28144cd9318488594317dc6"
23:00:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.336896 12283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal/instance:
23:00:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82b47e11f28144cd9318488594317dc6"
23:00:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.344367 12283 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.000s
23:00:40.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.348145 12297 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:40.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.349728 12283 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:00:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.349941 12283 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82b47e11f28144cd9318488594317dc6"
23:00:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.350273 12283 fs_report.cc:389] FS layout report
23:00:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/data
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.383502 12283 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.384626 12283 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.384979 12283 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.386150 12283 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:00:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.387972 12283 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:00:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.388108 12283 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:00:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.388231 12283 ts_tablet_manager.cc:616] Registered 0 tablets
23:00:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.388312 12283 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:00:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.425141 12283 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.65:45723
23:00:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.425208 12409 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.65:45723 every 8 connection(s)
23:00:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.426855 12283 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:00:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.435452 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12283
23:00:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.435745 12001 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal/instance
23:00:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.443634 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:0
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:00:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:00:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:00:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.457953 12410 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:00:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.458415 12410 heartbeater.cc:461] Registering TS with master...
23:00:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.459645 12410 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:00:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.462970 12185 ts_manager.cc:194] Registered new tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:00:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.465117 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.65:35073
23:00:40.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.470578 12411 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:00:40.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.470677 12412 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:00:40.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.470779 12411 heartbeater.cc:461] Registering TS with master...
23:00:40.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.470881 12412 heartbeater.cc:461] Registering TS with master...
23:00:40.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.471299 12411 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:00:40.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.471333 12412 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:00:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.472990 12042 ts_manager.cc:194] Registered new tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:00:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.473366 12114 ts_manager.cc:194] Registered new tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:00:40.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.805739 12419 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:40.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.806071 12419 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:40.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.806154 12419 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:40.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.814538 12419 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:40.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.814718 12419 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.66
23:00:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.825356 12419 tablet_server_runner.cc:78] Tablet server non-default flags:
23:00:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:0
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:00:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.827047 12419 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.828966 12419 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.839552 12426 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.839605 12425 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:40.839879 12428 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.840139 12419 server_base.cc:1061] running on GCE node
23:00:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.841401 12419 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:40.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.842648 12419 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.843902 12419 hybrid_clock.cc:648] HybridClock initialized: now 1783465240843821 us; error 76 us; skew 500 ppm
23:00:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.844225 12419 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:40.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.846820 12419 webserver.cc:534] Webserver started at http://127.11.184.66:37479/ using document root <none> and password file <none>
23:00:40.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.847890 12419 fs_manager.cc:362] Metadata directory not provided
23:00:40.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.847985 12419 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:40.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.848407 12419 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:00:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.850865 12419 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/instance:
23:00:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:00:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.851660 12419 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal/instance:
23:00:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:00:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:40.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.857198 12419 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
23:00:40.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.860931 12434 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:40.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.862628 12419 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.862813 12419 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.863054 12419 fs_report.cc:389] FS layout report
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/data
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.887045 12419 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.888132 12419 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.888427 12419 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:40.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.889495 12419 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:00:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.891353 12419 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:00:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.891571 12419 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:00:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.891682 12419 ts_tablet_manager.cc:616] Registered 0 tablets
23:00:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.891744 12419 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:00:40.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.931732 12419 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.66:32895
23:00:40.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.931784 12546 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.66:32895 every 8 connection(s)
23:00:40.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.933457 12419 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:00:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.935588 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12419
23:00:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.935793 12001 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal/instance
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.940845 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:0
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:00:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:00:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:00:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.953737 12548 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:00:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.953766 12549 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:00:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.954294 12548 heartbeater.cc:461] Registering TS with master...
23:00:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.954295 12549 heartbeater.cc:461] Registering TS with master...
23:00:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.954659 12547 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:00:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.954803 12547 heartbeater.cc:461] Registering TS with master...
23:00:40.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.955845 12547 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:00:40.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.956041 12548 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:00:40.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.956622 12549 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:00:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.957860 12185 ts_manager.cc:194] Registered new tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.957971 12042 ts_manager.cc:194] Registered new tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.958099 12114 ts_manager.cc:194] Registered new tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:40.958926 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.66:36081
23:00:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.304551 12554 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.304845 12554 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.304921 12554 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.313494 12554 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.313654 12554 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.67
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.324126 12554 tablet_server_runner.cc:78] Tablet server non-default flags:
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:0
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.325809 12554 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:41.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.328269 12554 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.339406 12560 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.339947 12562 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:41.339434 12559 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.340648 12554 server_base.cc:1061] running on GCE node
23:00:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.341320 12554 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.342379 12554 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:41.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.343654 12554 hybrid_clock.cc:648] HybridClock initialized: now 1783465241343601 us; error 31 us; skew 500 ppm
23:00:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.344058 12554 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.346306 12554 webserver.cc:534] Webserver started at http://127.11.184.67:41735/ using document root <none> and password file <none>
23:00:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.347178 12554 fs_manager.cc:362] Metadata directory not provided
23:00:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.347311 12554 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.347707 12554 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:00:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.350239 12554 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/instance:
23:00:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:00:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:41 on dist-test-slave-1pcn"
23:00:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.351168 12554 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal/instance:
23:00:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:00:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:41 on dist-test-slave-1pcn"
23:00:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.356969 12554 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.000s	sys 0.004s
23:00:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.360611 12568 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.362005 12554 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.362211 12554 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:41 on dist-test-slave-1pcn"
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.362468 12554 fs_report.cc:389] FS layout report
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/data
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.380373 12554 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.381350 12554 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.381675 12554 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.382715 12554 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:00:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.384479 12554 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:00:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.384605 12554 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:00:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.384723 12554 ts_tablet_manager.cc:616] Registered 0 tablets
23:00:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.384806 12554 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:00:41.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.419854 12554 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.67:46593
23:00:41.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.419890 12680 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.67:46593 every 8 connection(s)
23:00:41.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.421310 12554 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:00:41.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.424851 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12554
23:00:41.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.425027 12001 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal/instance
23:00:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.445456 12681 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:00:41.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.445479 12683 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:00:41.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.445531 12682 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:00:41.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.445955 12681 heartbeater.cc:461] Registering TS with master...
23:00:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.447013 12681 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:00:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.445957 12683 heartbeater.cc:461] Registering TS with master...
23:00:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.445960 12682 heartbeater.cc:461] Registering TS with master...
23:00:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.447780 12683 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:00:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.447886 12682 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:00:41.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.449416 12185 ts_manager.cc:194] Registered new tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:41.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.449564 12042 ts_manager.cc:194] Registered new tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:41.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.449759 12113 ts_manager.cc:194] Registered new tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:41.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.450605 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.67:60429
23:00:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.453022 12001 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
23:00:41.455 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:00:41.456 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

23:00:41.464 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.11.184.126"
      port: 46381
    }
    bound_http_address {
      host: "127.11.184.126"
      port: 42375
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.11.184.125"
      port: 41661
    }
    bound_http_address {
      host: "127.11.184.125"
      port: 45243
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.11.184.124"
      port: 42425
    }
    bound_http_address {
      host: "127.11.184.124"
      port: 35029
    }
  }
}

23:00:41.467 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

23:00:41.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.469195 12410 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:00:41.470 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.11.184.65"
      port: 45723
    }
    bound_http_address {
      host: "127.11.184.65"
      port: 40833
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.11.184.66"
      port: 32895
    }
    bound_http_address {
      host: "127.11.184.66"
      port: 37479
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.11.184.67"
      port: 46593
    }
    bound_http_address {
      host: "127.11.184.67"
      port: 41735
    }
  }
}

23:00:41.470 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:00:41.663 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
23:00:41.727 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
23:00:41.905 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.184.126:46381(127.11.184.126:46381)] connecting to peer
23:00:41.961 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.184.125:41661(127.11.184.125:41661)] connecting to peer
23:00:41.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:41.962070 12547 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:00:41.963 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.184.124:42425(127.11.184.124:42425)] connecting to peer
23:00:41.978 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.11.184.124:42425(127.11.184.124:42425)] Successfully connected to peer
23:00:41.979 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.11.184.125:41661(127.11.184.125:41661)] Successfully connected to peer
23:00:41.980 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.184.126:46381(127.11.184.126:46381)] Successfully connected to peer
23:00:42.069 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.11.184.126
23:00:42.069 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.11.184.125
23:00:42.079 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.11.184.124
23:00:42.281 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x74ee5f40, L:/127.0.0.1:54616 - R:/127.11.184.126:46381] using SASL/PLAIN
23:00:42.281 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x84201821, L:/127.0.0.1:39316 - R:/127.11.184.124:42425] using SASL/PLAIN
23:00:42.281 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x548e46ec, L:/127.0.0.1:59058 - R:/127.11.184.125:41661] using SASL/PLAIN
23:00:42.322 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
23:00:42.324 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=459), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.184.126:46381, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.184.125:41661, [94ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.184.124:42425, [429ms] Sub RPC ConnectToMaster: received response from server master-127.11.184.126:46381: OK, [429ms] Sub RPC ConnectToMaster: received response from server master-127.11.184.125:41661: OK, [430ms] Sub RPC ConnectToMaster: received response from server master-127.11.184.124:42425: OK, deferred=null) after lookup
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.369627 12185 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39316:
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849"
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:00:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:00:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:00:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:00:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:42.371815 12185 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
23:00:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.449260 12616 tablet_service.cc:1511] Processing CreateTablet for tablet d3889c79e97e44dfb205d9875fe68a97 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849 [id=b98f331fb0234df091e853d5b09d02b1]), partition=RANGE (key) PARTITION UNBOUNDED
23:00:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.448732 12482 tablet_service.cc:1511] Processing CreateTablet for tablet d3889c79e97e44dfb205d9875fe68a97 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849 [id=b98f331fb0234df091e853d5b09d02b1]), partition=RANGE (key) PARTITION UNBOUNDED
23:00:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.453855 12482 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3889c79e97e44dfb205d9875fe68a97. 1 dirs total, 0 dirs full, 0 dirs failed
23:00:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.455327 12345 tablet_service.cc:1511] Processing CreateTablet for tablet d3889c79e97e44dfb205d9875fe68a97 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849 [id=b98f331fb0234df091e853d5b09d02b1]), partition=RANGE (key) PARTITION UNBOUNDED
23:00:42.456 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table b98f331fb0234df091e853d5b09d02b1
23:00:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.456373 12681 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:00:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.459367 12345 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3889c79e97e44dfb205d9875fe68a97. 1 dirs total, 0 dirs full, 0 dirs failed
23:00:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.459630 12616 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3889c79e97e44dfb205d9875fe68a97. 1 dirs total, 0 dirs full, 0 dirs failed
23:00:42.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.465320 12713 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap starting.
23:00:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.480254 12713 tablet_bootstrap.cc:654] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Neither blocks nor log segments found. Creating new log.
23:00:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.481438 12713 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Log is configured to *not* fsync() on all Append() calls
23:00:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.488236 12714 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap starting.
23:00:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.491709 12716 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap starting.
23:00:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.491910 12713 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: No bootstrap required, opened a new log
23:00:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.492213 12713 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent bootstrapping tablet: real 0.027s	user 0.008s	sys 0.006s
23:00:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.498360 12714 tablet_bootstrap.cc:654] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Neither blocks nor log segments found. Creating new log.
23:00:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.499043 12716 tablet_bootstrap.cc:654] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Neither blocks nor log segments found. Creating new log.
23:00:42.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.500399 12714 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Log is configured to *not* fsync() on all Append() calls
23:00:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.504144 12716 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Log is configured to *not* fsync() on all Append() calls
23:00:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.503934 12713 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.504467 12713 raft_consensus.cc:385] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:00:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.504596 12713 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Initialized, Role: FOLLOWER
23:00:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.505782 12713 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.513408 12714 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: No bootstrap required, opened a new log
23:00:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.514191 12714 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent bootstrapping tablet: real 0.026s	user 0.010s	sys 0.008s
23:00:42.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.519143 12713 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent starting tablet: real 0.027s	user 0.017s	sys 0.002s
23:00:42.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.520339 12716 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: No bootstrap required, opened a new log
23:00:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.520906 12716 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent bootstrapping tablet: real 0.030s	user 0.009s	sys 0.011s
23:00:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.522718 12714 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.523494 12714 raft_consensus.cc:385] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:00:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.523749 12714 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Initialized, Role: FOLLOWER
23:00:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.524643 12714 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.528849 12716 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.529531 12716 raft_consensus.cc:385] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:00:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.529841 12716 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Initialized, Role: FOLLOWER
23:00:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.530841 12716 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.550165 12716 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent starting tablet: real 0.029s	user 0.009s	sys 0.016s
23:00:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.550360 12714 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent starting tablet: real 0.036s	user 0.015s	sys 0.013s
23:00:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.580968 12719 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:00:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.581290 12719 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.583729 12719 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:42.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.593477 12636 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" is_pre_election: true
23:00:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.593755 12365 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6" is_pre_election: true
23:00:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.594154 12636 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 0.
23:00:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.594270 12365 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 0.
23:00:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.595029 12437 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 
23:00:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.595499 12719 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:00:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.595671 12719 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:00:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.595786 12719 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 0 FOLLOWER]: Advancing to term 1
23:00:42.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.598070 12719 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.598780 12719 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 1 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.599349 12365 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6"
23:00:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.599407 12636 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:00:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.599591 12365 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 0 FOLLOWER]: Advancing to term 1
23:00:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.599638 12636 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 0 FOLLOWER]: Advancing to term 1
23:00:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.602201 12365 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 1.
23:00:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.602732 12435 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 
23:00:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.602857 12636 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 1.
23:00:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.603176 12719 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Leader election won for term 1
23:00:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.603705 12719 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 LEADER]: Becoming Leader. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Running, Role: LEADER
23:00:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.604323 12719 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.610214 12185 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 reported cstate change: term changed from 0 to 1, leader changed from <none> to fa076f9a47194d2daca748165b9575e2 (127.11.184.66). New cstate: current_term: 1 leader_uuid: "fa076f9a47194d2daca748165b9575e2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:00:42.653 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table b98f331fb0234df091e853d5b09d02b1
23:00:42.673 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:42.681046 12413 tablet.cc:2406] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:00:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:42.685197 12684 tablet.cc:2406] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:00:42.687 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299986, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:42.688201 12550 tablet.cc:2406] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:00:42.689 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=null) after lookup
23:00:42.698 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:42.700 [DEBUG - kudu-nio-3] (Connection.java:757) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] Successfully connected to peer
23:00:42.769 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x87d0fee8, L:/127.0.0.1:44888 - R:/127.11.184.66:32895] using TOKEN/null
23:00:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.791429 12636 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Refusing update from remote peer fa076f9a47194d2daca748165b9575e2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:00:42.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.793217 12719 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:00:42.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.794100 12365 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 1 FOLLOWER]: Refusing update from remote peer fa076f9a47194d2daca748165b9575e2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:00:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.797708 12719 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:00:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.825819 12728 mvcc.cc:204] Tried to move back new op lower bound from 7305073634459865088 to 7305073633712664576. Current Snapshot: MvccSnapshot[applied={T|T < 7305073634459865088}]
23:00:42.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.837924 12730 mvcc.cc:204] Tried to move back new op lower bound from 7305073634459865088 to 7305073633712664576. Current Snapshot: MvccSnapshot[applied={T|T < 7305073634459865088}]
23:00:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.848294 12729 mvcc.cc:204] Tried to move back new op lower bound from 7305073634459865088 to 7305073633712664576. Current Snapshot: MvccSnapshot[applied={T|T < 7305073634459865088}]
23:00:42.958 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:00:42.960 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.66:32895
23:00:42.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:00:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.962277 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12419
23:00:42.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:42.981694 12542 generic_service.cc:196] Checking for leaks (request via RPC)
23:00:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:43.086833 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12419
23:00:43.109 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:43.112 [DEBUG - kudu-nio-3] (Connection.java:239) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] handling channelInactive
23:00:43.113 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state READY due to: connection closed
23:00:43.113 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: connection closed
23:00:43.129 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:43.129 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:43.130 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test)) after lookup
23:00:43.130 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:43.133 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.133 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.133 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.148 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:43.149 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:43.149 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [20ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [23ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [23ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [38ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541))) after lookup
23:00:43.150 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:43.151 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.152 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.152 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.170 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:43.170 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:43.171 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [20ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [23ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [23ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [38ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [39ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [42ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [42ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [59ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453))) after lookup
23:00:43.171 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:43.173 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.174 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.174 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.209 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:43.210 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:43.210 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=100), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [20ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [23ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [23ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [38ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [39ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [42ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [42ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [59ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [61ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [64ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [64ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [95ms] refreshing cache from master, [95ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [99ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294))) after lookup
23:00:43.211 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:43.213 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.213 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.214 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.389 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:43.390 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:43.390 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=280), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [20ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [23ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [23ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [38ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [39ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [42ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [42ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [59ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [61ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [64ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [64ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [95ms] refreshing cache from master, [95ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [99ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [101ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [103ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [104ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [279ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294) -> (continuation of Deferred@1215227007 after retry RPC@931139864), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294) -> (continuation of Deferred@1215227007 after retry RPC@931139864))) after lookup
23:00:43.391 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:43.393 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:43.393 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:44.030 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:44.030 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:44.033 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=922), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [20ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [23ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [23ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [38ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [39ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [42ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [42ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [59ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [61ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [64ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [64ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [95ms] refreshing cache from master, [95ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [99ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [101ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [103ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [104ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [279ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [281ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [283ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [283ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [916ms] refreshing cache from master, [916ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [919ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294) -> (continuation of Deferred@1215227007 after retry RPC@931139864) -> (continuation of Deferred@1208874898 after retry RPC@2133993306), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294) -> (continuation of Deferred@1215227007 after retry RPC@931139864) -> (continuation of Deferred@1208874898 after retry RPC@2133993306))) after lookup
23:00:44.033 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:44.034 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:44.035 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:44.035 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.404518 12748 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:00:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.404770 12748 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.404829 12749 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:00:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.404990 12749 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.406453 12748 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.407161 12749 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:44.415948 12572 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:44.418459 12572 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:44.418447 12301 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.421442 12364 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6" is_pre_election: true
23:00:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.421713 12364 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 1.
23:00:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:44.421723 12301 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.422533 12569 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:00:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.423070 12748 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Leader pre-election won for term 2
23:00:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.423241 12748 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Starting leader election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:00:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.423349 12748 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 1 FOLLOWER]: Advancing to term 2
23:00:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.426698 12748 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.427496 12748 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 2 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.427546 12635 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" is_pre_election: true
23:00:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.427920 12635 raft_consensus.cc:2393] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82b47e11f28144cd9318488594317dc6 in current term 2: Already voted for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in this term.
23:00:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.428220 12364 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6"
23:00:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.428421 12364 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 1 FOLLOWER]: Advancing to term 2
23:00:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.428586 12300 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82b47e11f28144cd9318488594317dc6; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2
23:00:44.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:44.429822 12572 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.431926 12364 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 2.
23:00:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.432482 12749 raft_consensus.cc:2749] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:00:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.432626 12569 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:00:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.432994 12748 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Leader election won for term 2
23:00:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.433455 12748 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 LEADER]: Becoming Leader. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Running, Role: LEADER
23:00:44.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.434221 12748 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:44.438252 12185 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d reported cstate change: term changed from 1 to 2, leader changed from fa076f9a47194d2daca748165b9575e2 (127.11.184.66) to 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67). New cstate: current_term: 2 leader_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: HEALTHY } } }
23:00:44.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:44.939411 12572 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Couldn't send request to peer fa076f9a47194d2daca748165b9575e2. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:00:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:45.022706 12364 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Refusing update from remote peer 6bc03e9853ba4c04823c3c78a8a98d7d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
23:00:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:45.023526 12771 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Connected to new peer: Peer: permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
23:00:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:47.643513 12572 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Couldn't send request to peer fa076f9a47194d2daca748165b9575e2. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:00:48.009 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:48.009 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:48.010 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=4899), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [3ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [19ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [20ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [23ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [23ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [38ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [39ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [42ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [42ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [59ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [61ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [64ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [64ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [95ms] refreshing cache from master, [95ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [99ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [101ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [103ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [104ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [279ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [281ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [283ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [283ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [916ms] refreshing cache from master, [916ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [919ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [923ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [925ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [925ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [4895ms] refreshing cache from master, [4895ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [4899ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1898917443(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@2bfb2a22 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294) -> (continuation of Deferred@1215227007 after retry RPC@931139864) -> (continuation of Deferred@1208874898 after retry RPC@2133993306) -> (continuation of Deferred@1115123848 after retry RPC@747738987), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55991e5b -> wakeup thread Time-limited test -> (continuation of Deferred@1832267925 after retry RPC@215083541) -> (continuation of Deferred@1644455567 after retry RPC@410494453) -> (continuation of Deferred@1187744586 after retry RPC@681793294) -> (continuation of Deferred@1215227007 after retry RPC@931139864) -> (continuation of Deferred@1208874898 after retry RPC@2133993306) -> (continuation of Deferred@1115123848 after retry RPC@747738987))) after lookup
23:00:48.011 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:48.013 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] Successfully connected to peer
23:00:48.037 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x54848c36, L:/127.0.0.1:40552 - R:/127.11.184.67:46593] using TOKEN/null
23:00:48.067 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.66:32895
23:00:48.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.069993 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:00:48.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.420888 12797 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:48.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.421267 12797 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:48.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.421389 12797 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.431079 12797 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.431401 12797 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.66
23:00:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.442301 12797 tablet_server_runner.cc:78] Tablet server non-default flags:
23:00:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:48.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.444530 12797 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.446487 12797 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.457146 12805 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.457127 12803 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.457190 12802 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.457890 12797 server_base.cc:1061] running on GCE node
23:00:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.458560 12797 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.459604 12797 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.460865 12797 hybrid_clock.cc:648] HybridClock initialized: now 1783465248460818 us; error 28 us; skew 500 ppm
23:00:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.461339 12797 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:48.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.463954 12797 webserver.cc:534] Webserver started at http://127.11.184.66:37479/ using document root <none> and password file <none>
23:00:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.464811 12797 fs_manager.cc:362] Metadata directory not provided
23:00:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.464978 12797 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.471055 12797 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.001s
23:00:48.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.474182 12811 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.475736 12797 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
23:00:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.475924 12797 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:00:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.476752 12797 fs_report.cc:389] FS layout report
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/data
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.507869 12797 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.508868 12797 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.509233 12797 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.510386 12797 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:00:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.512851 12818 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:00:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.519372 12797 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:00:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.519521 12797 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:00:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.519671 12797 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:00:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.522759 12797 ts_tablet_manager.cc:616] Registered 1 tablets
23:00:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.522897 12797 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
23:00:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.523394 12818 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap starting.
23:00:48.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.545239 12818 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Log is configured to *not* fsync() on all Append() calls
23:00:48.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.564517 12818 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:00:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.565673 12818 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap complete.
23:00:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.567652 12818 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent bootstrapping tablet: real 0.045s	user 0.029s	sys 0.014s
23:00:48.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.571591 12797 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.66:32895
23:00:48.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.571713 12925 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.66:32895 every 8 connection(s)
23:00:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.573553 12797 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:00:48.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.577507 12818 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.579249 12818 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Initialized, Role: FOLLOWER
23:00:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.579341 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12797
23:00:48.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:48.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.580837 12818 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:48.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.619652 12818 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent starting tablet: real 0.052s	user 0.033s	sys 0.016s
23:00:48.632 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:00:48.633 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.67:46593
23:00:48.634 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:00:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.634761 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12554
23:00:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.638978 12926 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:00:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.639704 12926 heartbeater.cc:461] Registering TS with master...
23:00:48.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.641294 12926 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:00:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.646462 12185 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:48.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.648358 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.66:38007
23:00:48.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.650061 12931 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:00:48.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.650249 12931 heartbeater.cc:461] Registering TS with master...
23:00:48.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.650902 12931 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:00:48.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.651358 12927 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:00:48.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.651528 12927 heartbeater.cc:461] Registering TS with master...
23:00:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.652940 12113 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.653640 12927 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:00:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.653635 12926 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:00:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.658004 12043 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.661391 12676 generic_service.cc:196] Checking for leaks (request via RPC)
23:00:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:48.772744 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12554
23:00:48.803 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] handling channelInactive
23:00:48.803 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:48.804 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state READY due to: connection closed
23:00:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:48.804714 12943 negotiation.cc:336] Failed RPC negotiation. Trace:
23:00:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:48.775866 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.11.184.67:50689 (local address 127.11.184.66:32895)
23:00:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:48.776637 (+   771us) server_negotiation.cc:207] Beginning negotiation
23:00:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:48.776649 (+    12us) server_negotiation.cc:400] Waiting for connection header
23:00:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:48.802700 (+ 26051us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.11.184.67:50689: BlockingRecv error: recv got EOF from 127.11.184.67:50689 (error 108)
23:00:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":545,"thread_start_us":708,"threads_started":1}
23:00:48.805 [DEBUG - Time-limited test] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:48.806 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.806 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.806 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.829 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:48.829 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:48.830 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test)) after lookup
23:00:48.830 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:48.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.832 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.849 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:48.850 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:48.850 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [27ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [45ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042))) after lookup
23:00:48.851 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:48.852 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.852 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.852 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.868 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:48.869 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:48.869 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [27ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [45ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [48ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [48ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [64ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321))) after lookup
23:00:48.870 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:48.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.871 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.908 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:48.909 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:48.909 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=105), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [27ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [45ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [48ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [48ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [64ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [66ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [67ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [67ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [104ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887))) after lookup
23:00:48.910 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:48.911 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.911 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:48.912 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.169 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:49.169 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:49.171 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=367), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [27ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [45ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [48ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [48ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [64ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [66ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [67ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [67ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [104ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [105ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [107ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [108ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [361ms] refreshing cache from master, [361ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [364ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887) -> (continuation of Deferred@1965263252 after retry RPC@804334879), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887) -> (continuation of Deferred@1965263252 after retry RPC@804334879))) after lookup
23:00:49.172 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:49.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.173 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:49.867450 12944 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:00:49.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:49.867801 12944 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:49.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:49.869876 12944 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:49.870 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:49.870 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:49.871 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1066), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [27ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [45ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [48ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [48ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [64ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [66ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [67ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [67ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [104ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [105ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [107ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [108ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [361ms] refreshing cache from master, [361ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [364ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [367ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [369ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [369ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [1062ms] refreshing cache from master, [1062ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1065ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887) -> (continuation of Deferred@1965263252 after retry RPC@804334879) -> (continuation of Deferred@521104367 after retry RPC@967845060), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887) -> (continuation of Deferred@1965263252 after retry RPC@804334879) -> (continuation of Deferred@521104367 after retry RPC@967845060))) after lookup
23:00:49.872 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:00:49.874 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.875 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.875 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:00:49.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:49.877558 12814 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:00:49.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:49.879468 12814 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:00:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:49.882653 12364 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6" is_pre_election: true
23:00:49.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:49.883526 12812 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6
23:00:49.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:49.884066 12944 raft_consensus.cc:2749] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:00:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.157524 12950 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:00:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.157907 12950 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.158708 12950 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:50.159931 12300 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:00:50.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:50.161377 12300 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:00:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.168203 12881 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2" is_pre_election: true
23:00:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.168571 12881 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82b47e11f28144cd9318488594317dc6 in term 1.
23:00:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.169481 12301 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d
23:00:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.169827 12950 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Leader pre-election won for term 3
23:00:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.169932 12950 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:00:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.170001 12950 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 2 FOLLOWER]: Advancing to term 3
23:00:50.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.172076 12950 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:50.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.172636 12950 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 3 election: Requested vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.173280 12881 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2"
23:00:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.173492 12881 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 1 FOLLOWER]: Advancing to term 3
23:00:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:50.174268 12300 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:00:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.177492 12881 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82b47e11f28144cd9318488594317dc6 in term 3.
23:00:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.177955 12301 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d
23:00:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.178288 12950 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 FOLLOWER]: Leader election won for term 3
23:00:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.178789 12950 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 LEADER]: Becoming Leader. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Running, Role: LEADER
23:00:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.179378 12950 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:50.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.182989 12185 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 reported cstate change: term changed from 2 to 3, leader changed from 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67) to 82b47e11f28144cd9318488594317dc6 (127.11.184.65). New cstate: current_term: 3 leader_uuid: "82b47e11f28144cd9318488594317dc6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:00:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:50.625299 12300 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 -> Peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Couldn't send request to peer 6bc03e9853ba4c04823c3c78a8a98d7d. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:00:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.672659 12881 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Refusing update from remote peer 82b47e11f28144cd9318488594317dc6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
23:00:50.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:50.673460 12950 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
23:00:51.869 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:51.870 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:51.870 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3066), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [2ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [24ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [27ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [45ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [48ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [48ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [64ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [66ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [67ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [67ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [104ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [105ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [107ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [108ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [361ms] refreshing cache from master, [361ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [364ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [367ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [369ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [369ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [1062ms] refreshing cache from master, [1062ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1065ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [1067ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1071ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [1071ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [3061ms] refreshing cache from master, [3062ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [3065ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1079363287(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@61197567 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887) -> (continuation of Deferred@1965263252 after retry RPC@804334879) -> (continuation of Deferred@521104367 after retry RPC@967845060) -> (continuation of Deferred@1982350656 after retry RPC@285168902), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@377112d2 -> wakeup thread Time-limited test -> (continuation of Deferred@1485069546 after retry RPC@1742480042) -> (continuation of Deferred@465529254 after retry RPC@502317321) -> (continuation of Deferred@1417114793 after retry RPC@1326718887) -> (continuation of Deferred@1965263252 after retry RPC@804334879) -> (continuation of Deferred@521104367 after retry RPC@967845060) -> (continuation of Deferred@1982350656 after retry RPC@285168902))) after lookup
23:00:51.871 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:51.872 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] Successfully connected to peer
23:00:51.890 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x4c42f787, L:/127.0.0.1:51312 - R:/127.11.184.65:45723] using TOKEN/null
23:00:51.921 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.67:46593
23:00:51.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:51.923306 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:46593
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41735
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:00:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.264135 12988 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.264420 12988 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.264501 12988 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.273232 12988 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.273397 12988 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.67
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.283707 12988 tablet_server_runner.cc:78] Tablet server non-default flags:
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:46593
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41735
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.285465 12988 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:52.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.287412 12988 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.297811 12993 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.297847 12994 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.298686 12996 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.299558 12988 server_base.cc:1061] running on GCE node
23:00:52.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.300199 12988 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.301229 12988 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.302502 12988 hybrid_clock.cc:648] HybridClock initialized: now 1783465252302450 us; error 38 us; skew 500 ppm
23:00:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.302944 12988 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:52.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.305613 12988 webserver.cc:534] Webserver started at http://127.11.184.67:41735/ using document root <none> and password file <none>
23:00:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.306423 12988 fs_manager.cc:362] Metadata directory not provided
23:00:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.306581 12988 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.312305 12988 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.002s
23:00:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.315280 13002 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.316684 12988 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
23:00:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.316874 12988 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:00:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:41 on dist-test-slave-1pcn"
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.317744 12988 fs_report.cc:389] FS layout report
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/data
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.340149 12988 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.341182 12988 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.341542 12988 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:52.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.342803 12988 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:00:52.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.345146 13009 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:00:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.351193 12988 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:00:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.351281 12988 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:00:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.351382 12988 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:00:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.354307 12988 ts_tablet_manager.cc:616] Registered 1 tablets
23:00:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.354399 12988 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
23:00:52.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.354909 13009 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap starting.
23:00:52.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.377334 13009 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Log is configured to *not* fsync() on all Append() calls
23:00:52.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.395845 12988 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.67:46593
23:00:52.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.395884 13116 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.67:46593 every 8 connection(s)
23:00:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.397768 12988 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:00:52.401 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.400650 13009 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:00:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.400880 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 12988
23:00:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.401458 13009 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap complete.
23:00:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.403436 13009 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent bootstrapping tablet: real 0.049s	user 0.023s	sys 0.025s
23:00:52.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.413275 13009 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:52.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.414659 13009 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Initialized, Role: FOLLOWER
23:00:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.415688 13009 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:52.445 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:00:52.447 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.65:45723
23:00:52.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:00:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.448000 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12283
23:00:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.456949 13009 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent starting tablet: real 0.053s	user 0.024s	sys 0.028s
23:00:52.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.470166 12405 generic_service.cc:196] Checking for leaks (request via RPC)
23:00:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.472621 13117 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:00:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.473076 13117 heartbeater.cc:461] Registering TS with master...
23:00:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.474192 13119 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:00:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.474325 13119 heartbeater.cc:461] Registering TS with master...
23:00:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.474383 13117 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:00:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.474866 13119 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:00:52.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.477896 12113 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:52.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.479147 12184 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.480988 13118 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:00:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.481077 12184 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.67:54727
23:00:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.481158 13118 heartbeater.cc:461] Registering TS with master...
23:00:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.481712 13118 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:00:52.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.483188 12043 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:52.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.483982 13117 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:00:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:52.579635 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12283
23:00:52.600 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] handling channelInactive
23:00:52.600 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state READY due to: connection closed
23:00:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:52.600883 13132 negotiation.cc:336] Failed RPC negotiation. Trace:
23:00:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:52.580793 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.11.184.65:53451 (local address 127.11.184.67:46593)
23:00:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:52.581426 (+   633us) server_negotiation.cc:207] Beginning negotiation
23:00:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:52.581438 (+    12us) server_negotiation.cc:400] Waiting for connection header
23:00:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:00:52.599291 (+ 17853us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.11.184.65:53451: BlockingRecv error: recv got EOF from 127.11.184.65:53451 (error 108)
23:00:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":440,"thread_start_us":279,"threads_started":1}
23:00:52.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:52.602 [DEBUG - Time-limited test] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:52.604 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.604 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.604 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.628 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:52.629 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:52.629 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test)) after lookup
23:00:52.630 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:52.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.631 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.649 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:52.649 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:52.649 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731))) after lookup
23:00:52.650 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:52.651 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.651 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.651 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.689 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:52.689 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:52.689 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [47ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [49ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [49ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [86ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032))) after lookup
23:00:52.690 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:52.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.691 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.769 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:52.770 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:52.770 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=168), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [47ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [49ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [49ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [86ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [87ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [89ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [89ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774))) after lookup
23:00:52.771 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:52.772 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.773 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:52.773 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.248 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:53.248 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:53.249 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=646), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [47ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [49ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [49ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [86ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [87ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [89ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [89ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [169ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [171ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [171ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [643ms] refreshing cache from master, [643ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [646ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313))) after lookup
23:00:53.249 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:53.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.251 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.589 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:53.589 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:53.589 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=987), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [47ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [49ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [49ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [86ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [87ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [89ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [89ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [169ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [171ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [171ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [643ms] refreshing cache from master, [643ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [646ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [647ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [648ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [649ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [984ms] refreshing cache from master, [984ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [986ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313) -> (continuation of Deferred@1731948108 after retry RPC@1072407997), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313) -> (continuation of Deferred@1731948108 after retry RPC@1072407997))) after lookup
23:00:53.590 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:53.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.592 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.708 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:53.709 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:00:53.709 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1106), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [47ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [49ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [49ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [86ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [87ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [89ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [89ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [169ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [171ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [171ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [643ms] refreshing cache from master, [643ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [646ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [647ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [648ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [649ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [984ms] refreshing cache from master, [984ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [986ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [988ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [989ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [990ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1103ms] refreshing cache from master, [1103ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1106ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313) -> (continuation of Deferred@1731948108 after retry RPC@1072407997) -> (continuation of Deferred@2019914527 after retry RPC@440530915), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313) -> (continuation of Deferred@1731948108 after retry RPC@1072407997) -> (continuation of Deferred@2019914527 after retry RPC@440530915))) after lookup
23:00:53.710 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:00:53.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.712 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:00:53.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.845326 13133 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 82b47e11f28144cd9318488594317dc6)
23:00:53.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.845572 13133 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:53.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.846366 13133 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:53.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:53.849287 12812 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:00:53.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.855026 13072 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" is_pre_election: true
23:00:53.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.855660 13072 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 2.
23:00:53.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.856653 12814 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2; no voters: 82b47e11f28144cd9318488594317dc6
23:00:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.857005 13133 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Leader pre-election won for term 4
23:00:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.857146 13133 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 82b47e11f28144cd9318488594317dc6)
23:00:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.857275 13133 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 3 FOLLOWER]: Advancing to term 4
23:00:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.859467 13133 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.860157 13133 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 4 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:00:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.860641 13072 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:00:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.860797 13072 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 2 FOLLOWER]: Advancing to term 4
23:00:53.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:53.861891 12812 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:00:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.864711 13072 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 4.
23:00:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.865154 12814 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2; no voters: 82b47e11f28144cd9318488594317dc6
23:00:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.865474 13133 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 4 FOLLOWER]: Leader election won for term 4
23:00:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.865943 13133 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 4 LEADER]: Becoming Leader. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Running, Role: LEADER
23:00:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.866506 13133 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:53.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:53.870342 12184 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 reported cstate change: term changed from 3 to 4, leader changed from 82b47e11f28144cd9318488594317dc6 (127.11.184.65) to fa076f9a47194d2daca748165b9575e2 (127.11.184.66). New cstate: current_term: 4 leader_uuid: "fa076f9a47194d2daca748165b9575e2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:00:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:54.308377 12812 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 -> Peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Couldn't send request to peer 82b47e11f28144cd9318488594317dc6. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:00:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:54.414427 13072 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Refusing update from remote peer fa076f9a47194d2daca748165b9575e2: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
23:00:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:54.415506 13133 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
23:00:55.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:55.433332 12812 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111) [suppressed 4 similar messages]
23:00:56.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:56.961043 12812 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 -> Peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Couldn't send request to peer 82b47e11f28144cd9318488594317dc6. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:00:57.009 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:57.009 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:57.009 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=16, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=4407), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [26ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [27ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [29ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [46ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [47ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [49ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [49ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [86ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [87ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [89ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [89ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [169ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [171ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [171ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [643ms] refreshing cache from master, [643ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [646ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [647ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [648ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [649ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [984ms] refreshing cache from master, [984ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [986ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [988ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [989ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [990ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1103ms] refreshing cache from master, [1103ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1106ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [1108ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1109ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1110ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4403ms] refreshing cache from master, [4404ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [4406ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@734160207(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@73ce666a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313) -> (continuation of Deferred@1731948108 after retry RPC@1072407997) -> (continuation of Deferred@2019914527 after retry RPC@440530915) -> (continuation of Deferred@1055099887 after retry RPC@205359066), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@384e9727 -> wakeup thread Time-limited test -> (continuation of Deferred@333281392 after retry RPC@1608921731) -> (continuation of Deferred@409865228 after retry RPC@1987205032) -> (continuation of Deferred@1853031863 after retry RPC@173592774) -> (continuation of Deferred@77609467 after retry RPC@1211242313) -> (continuation of Deferred@1731948108 after retry RPC@1072407997) -> (continuation of Deferred@2019914527 after retry RPC@440530915) -> (continuation of Deferred@1055099887 after retry RPC@205359066))) after lookup
23:00:57.010 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:57.011 [DEBUG - kudu-nio-3] (Connection.java:757) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] Successfully connected to peer
23:00:57.036 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xa78395db, L:/127.0.0.1:39552 - R:/127.11.184.66:32895] using TOKEN/null
23:00:57.061 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.65:45723
23:00:57.061 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.061933 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:45723
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40833
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:00:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:57.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:57.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:57.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:00:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.416364 13178 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:00:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.416687 13178 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:00:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.416841 13178 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:00:57.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.425542 13178 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:00:57.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.425714 13178 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.65
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.436971 13178 tablet_server_runner.cc:78] Tablet server non-default flags:
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:00:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:45723
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40833
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:00:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:00:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.439278 13178 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:00:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.441537 13178 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:00:57.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.452304 13183 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:57.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.452732 13186 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:57.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:57.452256 13184 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:00:57.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.453945 13178 server_base.cc:1061] running on GCE node
23:00:57.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.454561 13178 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:00:57.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.455608 13178 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:00:57.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.456804 13178 hybrid_clock.cc:648] HybridClock initialized: now 1783465257456744 us; error 55 us; skew 500 ppm
23:00:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.457113 13178 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:00:57.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.459830 13178 webserver.cc:534] Webserver started at http://127.11.184.65:40833/ using document root <none> and password file <none>
23:00:57.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.460690 13178 fs_manager.cc:362] Metadata directory not provided
23:00:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.460850 13178 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:00:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.466800 13178 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
23:00:57.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.470038 13192 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:00:57.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.471676 13178 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
23:00:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.471875 13178 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82b47e11f28144cd9318488594317dc6"
23:00:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.472700 13178 fs_report.cc:389] FS layout report
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/data
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:00:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:00:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.505342 13178 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:00:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.506423 13178 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:00:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.506810 13178 kserver.cc:163] Server-wide thread pool size limit: 3276
23:00:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.507972 13178 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:00:57.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.510389 13199 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:00:57.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.516687 13178 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:00:57.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.516834 13178 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:00:57.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.516995 13178 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:00:57.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.520552 13178 ts_tablet_manager.cc:616] Registered 1 tablets
23:00:57.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.520689 13178 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.003s
23:00:57.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.521278 13199 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap starting.
23:00:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.546185 13199 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Log is configured to *not* fsync() on all Append() calls
23:00:57.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.569669 13199 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:00:57.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.570350 13199 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap complete.
23:00:57.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.571916 13199 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent bootstrapping tablet: real 0.051s	user 0.027s	sys 0.023s
23:00:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.575304 13178 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.65:45723
23:00:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.575366 13306 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.65:45723 every 8 connection(s)
23:00:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.577068 13178 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:00:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.579855 13199 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.580960 13199 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Initialized, Role: FOLLOWER
23:00:57.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:57.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.583469 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 13178
23:00:57.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.592748 13199 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:57.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.603094 13199 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent starting tablet: real 0.031s	user 0.009s	sys 0.005s
23:00:57.612 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:00:57.619 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.66:32895
23:00:57.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:00:57.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.620618 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12797
23:00:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.637667 12921 generic_service.cc:196] Checking for leaks (request via RPC)
23:00:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.640513 13308 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:00:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.640707 13311 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:00:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.641009 13311 heartbeater.cc:461] Registering TS with master...
23:00:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.641103 13307 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:00:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.641240 13307 heartbeater.cc:461] Registering TS with master...
23:00:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.642222 13311 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.643929 13307 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.641017 13308 heartbeater.cc:461] Registering TS with master...
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.645149 13308 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.645780 12114 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.646689 12185 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.648738 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.65:39441
23:00:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.650111 12042 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:00:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.652952 13307 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:00:57.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:57.753885 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12797
23:00:57.774 [DEBUG - kudu-nio-3] (Connection.java:239) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] handling channelInactive
23:00:57.775 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state READY due to: connection closed
23:00:57.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:00:57.776 [DEBUG - Time-limited test] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:57.778 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.778 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.778 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.788 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:57.789 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:57.789 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [12ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1665688976(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3ab2d74 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@612dac1a -> wakeup thread Time-limited test)) after lookup
23:00:57.790 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:57.791 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.791 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.791 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.809 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:57.809 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:57.810 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [12ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [13ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [33ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1665688976(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3ab2d74 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@612dac1a -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073))) after lookup
23:00:57.811 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:57.812 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.812 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.812 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.828 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:57.828 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:57.829 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [12ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [13ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [33ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [34ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [36ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [36ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [52ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1665688976(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3ab2d74 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@612dac1a -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180))) after lookup
23:00:57.829 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:57.830 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.830 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.830 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.868 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:57.868 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:57.869 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=93), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [12ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [13ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [33ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [34ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [36ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [36ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [52ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [53ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [54ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [54ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [92ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1665688976(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3ab2d74 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180) -> (continuation of Deferred@914841794 after retry RPC@675781762), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@612dac1a -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180) -> (continuation of Deferred@914841794 after retry RPC@675781762))) after lookup
23:00:57.870 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:57.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:57.871 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:58.048 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:00:58.049 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:00:58.049 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=273), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [12ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [13ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [33ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [34ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [36ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [36ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [52ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [53ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [54ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [54ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [92ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [93ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [95ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [95ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [272ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1665688976(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3ab2d74 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180) -> (continuation of Deferred@914841794 after retry RPC@675781762) -> (continuation of Deferred@790466359 after retry RPC@992650714), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@612dac1a -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180) -> (continuation of Deferred@914841794 after retry RPC@675781762) -> (continuation of Deferred@790466359 after retry RPC@992650714))) after lookup
23:00:58.050 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:00:58.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:58.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:58.051 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:00:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.939733 13322 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Starting pre-election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:00:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.939916 13322 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:58.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.941547 13322 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:58.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:58.942683 13006 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:58.944186 13006 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:58.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.950871 13262 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6" is_pre_election: true
23:00:58.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.951463 13262 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 3.
23:00:58.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.952538 13003 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:00:58.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.952984 13322 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Leader pre-election won for term 5
23:00:58.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.953102 13322 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Starting leader election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:00:58.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.953179 13322 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 4 FOLLOWER]: Advancing to term 5
23:00:58.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.996330 13322 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:58.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.997078 13322 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 5 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:00:58.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.997756 13262 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6"
23:00:58.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:58.998052 13262 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 3 FOLLOWER]: Advancing to term 5
23:00:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:58.998829 13006 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:00:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.096414 13262 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 5.
23:00:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.097888 13003 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:00:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.098563 13322 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 FOLLOWER]: Leader election won for term 5
23:00:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.099118 13322 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 LEADER]: Becoming Leader. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Running, Role: LEADER
23:00:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.099968 13322 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:00:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.104645 12185 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d reported cstate change: term changed from 4 to 5, leader changed from fa076f9a47194d2daca748165b9575e2 (127.11.184.66) to 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67). New cstate: current_term: 5 leader_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: HEALTHY } } }
23:00:59.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:00:59.528069 13006 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Couldn't send request to peer fa076f9a47194d2daca748165b9575e2. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:00:59.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.573030 13262 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Refusing update from remote peer 6bc03e9853ba4c04823c3c78a8a98d7d: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
23:00:59.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:00:59.574050 13322 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Connected to new peer: Peer: permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
23:01:00.028 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:00.029 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:00.030 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2254), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [12ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [13ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [33ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [34ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [36ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [36ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [52ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [53ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [54ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [54ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [92ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [93ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [95ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [95ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [272ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [273ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [275ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [275ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2249ms] refreshing cache from master, [2249ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [2252ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1665688976(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3ab2d74 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180) -> (continuation of Deferred@914841794 after retry RPC@675781762) -> (continuation of Deferred@790466359 after retry RPC@992650714) -> (continuation of Deferred@637558114 after retry RPC@1223309325), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@612dac1a -> wakeup thread Time-limited test -> (continuation of Deferred@1799403935 after retry RPC@1728157073) -> (continuation of Deferred@221881631 after retry RPC@1719600180) -> (continuation of Deferred@914841794 after retry RPC@675781762) -> (continuation of Deferred@790466359 after retry RPC@992650714) -> (continuation of Deferred@637558114 after retry RPC@1223309325))) after lookup
23:01:00.030 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:00.031 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] Successfully connected to peer
23:01:00.047 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x02f4bff8, L:/127.0.0.1:37340 - R:/127.11.184.67:46593] using TOKEN/null
23:01:00.072 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.66:32895
23:01:00.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.073771 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:01:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.466710 13353 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.467027 13353 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.467113 13353 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.476151 13353 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.476328 13353 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.66
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.487277 13353 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:01:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.489001 13353 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:00.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.490903 13353 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:00.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.501538 13359 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.501564 13358 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:00.502328 13361 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.503592 13353 server_base.cc:1061] running on GCE node
23:01:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.504266 13353 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:00.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.505350 13353 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.506585 13353 hybrid_clock.cc:648] HybridClock initialized: now 1783465260506511 us; error 64 us; skew 500 ppm
23:01:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.506992 13353 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.509366 13353 webserver.cc:534] Webserver started at http://127.11.184.66:37479/ using document root <none> and password file <none>
23:01:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.510557 13353 fs_manager.cc:362] Metadata directory not provided
23:01:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.510779 13353 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.516666 13353 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
23:01:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.519852 13367 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:00.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.521353 13353 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.521544 13353 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.522298 13353 fs_report.cc:389] FS layout report
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/data
23:01:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.555302 13353 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.556380 13353 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.556789 13353 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:00.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.558058 13353 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:00.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.560778 13374 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:00.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.567760 13353 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:00.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.567906 13353 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
23:01:00.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.568051 13353 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.571676 13353 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.571812 13353 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.004s	sys 0.000s
23:01:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.572312 13374 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap starting.
23:01:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.596969 13374 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Log is configured to *not* fsync() on all Append() calls
23:01:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.624706 13374 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.625913 13374 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap complete.
23:01:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.627286 13374 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent bootstrapping tablet: real 0.055s	user 0.039s	sys 0.015s
23:01:00.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.634840 13374 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:00.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.635743 13374 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Initialized, Role: FOLLOWER
23:01:00.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.636703 13374 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.638394 13374 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent starting tablet: real 0.011s	user 0.009s	sys 0.000s
23:01:00.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.641371 13353 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.66:32895
23:01:00.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.641462 13482 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.66:32895 every 8 connection(s)
23:01:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.643023 13353 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.644954 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 13353
23:01:00.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:00.708 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:00.709 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.67:46593
23:01:00.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:01:00.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.710388 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12988
23:01:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.710920 13485 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.711428 13485 heartbeater.cc:461] Registering TS with master...
23:01:00.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.712647 13485 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:00.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.716259 12114 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:00.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.722010 13483 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:00.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.722546 13483 heartbeater.cc:461] Registering TS with master...
23:01:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.722906 13484 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.723101 13484 heartbeater.cc:461] Registering TS with master...
23:01:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.723249 13483 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.723732 13484 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:00.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.724798 12185 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.725399 12042 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.726352 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.66:38557
23:01:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.732180 13437 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 4 FOLLOWER]: Advancing to term 5
23:01:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.736779 13112 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.802836 13483 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:00.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:00.845002 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12988
23:01:00.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:00.866 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] handling channelInactive
23:01:00.866 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state READY due to: connection closed
23:01:00.886 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:00.887 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.887 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.888 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.908 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:00.908 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:00.909 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=3, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [22ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [42ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1303239742(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@562aa7a0 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@541f1b84 -> wakeup thread Time-limited test)) after lookup
23:01:00.909 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:00.910 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.911 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.928 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:00.928 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:00.928 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=5, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [22ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [42ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [43ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [45ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [61ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1303239742(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@562aa7a0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@541f1b84 -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835))) after lookup
23:01:00.929 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:00.930 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.930 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.930 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.969 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:00.969 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:00.969 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=7, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=102), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [22ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [42ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [43ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [45ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [61ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [63ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [64ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [102ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1303239742(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@562aa7a0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@541f1b84 -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025))) after lookup
23:01:00.970 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:00.971 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.971 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:00.971 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.149 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:01.149 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:01.150 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=9, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=283), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [22ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [42ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [43ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [45ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [61ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [63ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [64ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [102ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [103ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [105ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [105ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [279ms] refreshing cache from master, [279ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [282ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1303239742(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@562aa7a0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025) -> (continuation of Deferred@1891193982 after retry RPC@1415323213), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@541f1b84 -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025) -> (continuation of Deferred@1891193982 after retry RPC@1415323213))) after lookup
23:01:01.150 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:01.151 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.152 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.152 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.810 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:01.810 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:01.810 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=11, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=944), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [22ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [42ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [43ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [45ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [61ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [63ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [64ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [102ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [103ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [105ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [105ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [279ms] refreshing cache from master, [279ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [282ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [284ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [286ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [286ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [939ms] refreshing cache from master, [940ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [943ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1303239742(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@562aa7a0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025) -> (continuation of Deferred@1891193982 after retry RPC@1415323213) -> (continuation of Deferred@416154292 after retry RPC@752146838), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@541f1b84 -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025) -> (continuation of Deferred@1891193982 after retry RPC@1415323213) -> (continuation of Deferred@416154292 after retry RPC@752146838))) after lookup
23:01:01.811 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:01.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.812 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:01.999517 13504 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:01:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:01.999707 13504 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:02.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.001406 13504 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:02.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:02.012648 13195 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:02.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:02.015007 13195 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.018960 13437 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2" is_pre_election: true
23:01:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.020064 13196 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82b47e11f28144cd9318488594317dc6; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2
23:01:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.020542 13504 raft_consensus.cc:2749] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
23:01:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.301113 13509 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:01:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.301467 13509 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.303359 13509 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:02.305390 13370 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:02.307403 13370 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:02.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.311395 13262 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6" is_pre_election: true
23:01:02.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.311671 13262 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 5.
23:01:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.312222 13368 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d
23:01:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.312608 13509 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 5 FOLLOWER]: Leader pre-election won for term 6
23:01:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.312768 13509 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:01:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.312867 13509 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 5 FOLLOWER]: Advancing to term 6
23:01:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.314899 13509 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.315723 13509 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 6 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.316202 13262 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6"
23:01:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.316402 13262 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 5 FOLLOWER]: Advancing to term 6
23:01:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:02.317375 13370 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.319463 13262 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 6.
23:01:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.319921 13368 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d
23:01:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.320256 13509 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Leader election won for term 6
23:01:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.320808 13509 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 LEADER]: Becoming Leader. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Running, Role: LEADER
23:01:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.321378 13509 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.325183 12185 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 reported cstate change: term changed from 5 to 6, leader changed from 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67) to fa076f9a47194d2daca748165b9575e2 (127.11.184.66). New cstate: current_term: 6 leader_uuid: "fa076f9a47194d2daca748165b9575e2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:01:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:02.727399 13370 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 -> Peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Couldn't send request to peer 6bc03e9853ba4c04823c3c78a8a98d7d. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:01:02.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.861109 13262 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Refusing update from remote peer fa076f9a47194d2daca748165b9575e2: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
23:01:02.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:02.861992 13509 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 20, Time since last communication: 0.000s
23:01:03.148 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:03.148 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:03.149 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=13, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2282), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [22ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [42ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [43ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [45ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [61ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [63ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [64ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [102ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [103ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [105ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [105ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [279ms] refreshing cache from master, [279ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [282ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [284ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [286ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [286ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [939ms] refreshing cache from master, [940ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [943ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [945ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [946ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [946ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [2279ms] refreshing cache from master, [2279ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [2282ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1303239742(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@562aa7a0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025) -> (continuation of Deferred@1891193982 after retry RPC@1415323213) -> (continuation of Deferred@416154292 after retry RPC@752146838) -> (continuation of Deferred@796846683 after retry RPC@627875403), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@541f1b84 -> wakeup thread Time-limited test -> (continuation of Deferred@1202297729 after retry RPC@1808165835) -> (continuation of Deferred@954741204 after retry RPC@1621663025) -> (continuation of Deferred@1891193982 after retry RPC@1415323213) -> (continuation of Deferred@416154292 after retry RPC@752146838) -> (continuation of Deferred@796846683 after retry RPC@627875403))) after lookup
23:01:03.149 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:03.150 [DEBUG - kudu-nio-1] (Connection.java:757) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] Successfully connected to peer
23:01:03.166 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x84a0c9db, L:/127.0.0.1:46264 - R:/127.11.184.66:32895] using TOKEN/null
23:01:03.194 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.67:46593
23:01:03.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.196813 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:46593
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41735
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.540448 13535 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.540850 13535 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.540982 13535 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.549738 13535 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.549986 13535 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.67
23:01:03.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.560459 13535 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:03.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:46593
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41735
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.562633 13535 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.564689 13535 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.575271 13541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:03.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.575301 13540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:03.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.577083 13543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:03.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.577845 13535 server_base.cc:1061] running on GCE node
23:01:03.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.578452 13535 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:03.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.579644 13535 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:03.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.580911 13535 hybrid_clock.cc:648] HybridClock initialized: now 1783465263580836 us; error 41 us; skew 500 ppm
23:01:03.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.581360 13535 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.584322 13535 webserver.cc:534] Webserver started at http://127.11.184.67:41735/ using document root <none> and password file <none>
23:01:03.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.585249 13535 fs_manager.cc:362] Metadata directory not provided
23:01:03.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.585407 13535 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.591760 13535 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
23:01:03.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.595094 13549 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.596773 13535 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:01:03.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.596976 13535 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:03.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:01:03.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:41 on dist-test-slave-1pcn"
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.597752 13535 fs_report.cc:389] FS layout report
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/data
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:03.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.624471 13535 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:03.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.625447 13535 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.625876 13535 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:03.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.626988 13535 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.629359 13556 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:03.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.635396 13535 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:03.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.635509 13535 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:01:03.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.635725 13535 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.638762 13535 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:03.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.638890 13535 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
23:01:03.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.639318 13556 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap starting.
23:01:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.666687 13556 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Log is configured to *not* fsync() on all Append() calls
23:01:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.689500 13535 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.67:46593
23:01:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.689548 13663 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.67:46593 every 8 connection(s)
23:01:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.691238 13535 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:01:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.695919 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 13535
23:01:03.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:03.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.699424 13556 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.700152 13556 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap complete.
23:01:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.701920 13556 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent bootstrapping tablet: real 0.063s	user 0.038s	sys 0.019s
23:01:03.710 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:03.711 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.66:32895
23:01:03.711 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:01:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.711850 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 13353
23:01:03.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.711563 13556 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:03.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.712782 13556 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Initialized, Role: FOLLOWER
23:01:03.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.713640 13556 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.778518 13556 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent starting tablet: real 0.076s	user 0.027s	sys 0.050s
23:01:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.782668 13664 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.783052 13664 heartbeater.cc:461] Registering TS with master...
23:01:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.783967 13664 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.787381 12184 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.788843 12184 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.67:58439
23:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.789390 13665 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.789556 13665 heartbeater.cc:461] Registering TS with master...
23:01:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.790185 13665 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.792150 12042 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.793853 13666 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.793993 13666 heartbeater.cc:461] Registering TS with master...
23:01:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.794287 13666 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.795656 13478 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.793891 13664 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.797142 12114 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:03.904207 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 13353
23:01:03.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:03.923084 12027 connection.cc:573] server connection from 127.11.184.66:57697 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:01:03.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:03.924 [DEBUG - kudu-nio-1] (Connection.java:239) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] handling channelInactive
23:01:03.924 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state READY due to: connection closed
23:01:03.925 [DEBUG - Time-limited test] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:03.926 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.926 [DEBUG - kudu-nio-2] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.926 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.949 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:03.949 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:03.950 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [25ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@909105915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@59cc8c9e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ee2da9b -> wakeup thread Time-limited test)) after lookup
23:01:03.950 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:03.952 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.952 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.952 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.969 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:03.969 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:03.970 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [25ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [28ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [28ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [44ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@909105915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@59cc8c9e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ee2da9b -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152))) after lookup
23:01:03.970 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:03.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.971 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:03.971 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.009 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:04.009 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:04.009 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [25ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [28ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [28ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [44ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [47ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [47ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [84ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@909105915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@59cc8c9e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ee2da9b -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204))) after lookup
23:01:04.010 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:04.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.011 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.012 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.109 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:04.109 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:04.110 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=185), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [25ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [28ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [28ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [44ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [47ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [47ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [84ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [86ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [87ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [88ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [181ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [185ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@909105915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@59cc8c9e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204) -> (continuation of Deferred@1519266844 after retry RPC@1821215369), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ee2da9b -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204) -> (continuation of Deferred@1519266844 after retry RPC@1821215369))) after lookup
23:01:04.110 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:04.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.111 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.549 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:04.549 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:04.550 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=625), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [25ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [28ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [28ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [44ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [47ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [47ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [84ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [86ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [87ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [88ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [181ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [185ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [186ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [187ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [187ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [621ms] refreshing cache from master, [621ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [625ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@909105915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@59cc8c9e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204) -> (continuation of Deferred@1519266844 after retry RPC@1821215369) -> (continuation of Deferred@77410860 after retry RPC@79968756), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ee2da9b -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204) -> (continuation of Deferred@1519266844 after retry RPC@1821215369) -> (continuation of Deferred@77410860 after retry RPC@79968756))) after lookup
23:01:04.550 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:04.551 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.551 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:04.551 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.208132 13679 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:01:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.208316 13679 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.209057 13679 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:05.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:05.211584 13196 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:01:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.217707 13619 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" is_pre_election: true
23:01:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.218186 13619 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82b47e11f28144cd9318488594317dc6 in term 5.
23:01:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.219096 13195 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:01:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.219491 13679 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Leader pre-election won for term 7
23:01:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.219632 13679 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Starting leader election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:01:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.219730 13679 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 6 FOLLOWER]: Advancing to term 7
23:01:05.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.221879 13679 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:05.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.222617 13679 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 7 election: Requested vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:05.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.223210 13619 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:01:05.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.223431 13619 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 5 FOLLOWER]: Advancing to term 7
23:01:05.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:05.224138 13196 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:01:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.227259 13619 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82b47e11f28144cd9318488594317dc6 in term 7.
23:01:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.227692 13195 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:01:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.228001 13679 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Leader election won for term 7
23:01:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.228438 13679 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 LEADER]: Becoming Leader. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Running, Role: LEADER
23:01:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.229194 13679 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.232909 12184 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 reported cstate change: term changed from 6 to 7, leader changed from fa076f9a47194d2daca748165b9575e2 (127.11.184.66) to 82b47e11f28144cd9318488594317dc6 (127.11.184.65). New cstate: current_term: 7 leader_uuid: "82b47e11f28144cd9318488594317dc6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:01:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:05.775010 13196 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 -> Peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Couldn't send request to peer fa076f9a47194d2daca748165b9575e2. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:01:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.816891 13619 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Refusing update from remote peer 82b47e11f28144cd9318488594317dc6: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
23:01:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.817845 13697 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
23:01:05.889 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:05.889 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:05.889 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1965), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [2ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [2ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [25ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [26ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [28ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [28ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [44ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [46ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [47ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [47ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [84ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [86ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [87ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [88ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [181ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [185ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [186ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [187ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [187ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [621ms] refreshing cache from master, [621ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [625ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [626ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [627ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [627ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [1962ms] refreshing cache from master, [1962ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1964ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@909105915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@59cc8c9e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204) -> (continuation of Deferred@1519266844 after retry RPC@1821215369) -> (continuation of Deferred@77410860 after retry RPC@79968756) -> (continuation of Deferred@1531544599 after retry RPC@2120686867), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ee2da9b -> wakeup thread Time-limited test -> (continuation of Deferred@396619165 after retry RPC@1338887152) -> (continuation of Deferred@138424431 after retry RPC@695466204) -> (continuation of Deferred@1519266844 after retry RPC@1821215369) -> (continuation of Deferred@77410860 after retry RPC@79968756) -> (continuation of Deferred@1531544599 after retry RPC@2120686867))) after lookup
23:01:05.890 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:05.891 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] Successfully connected to peer
23:01:05.909 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdf1c6f60, L:/127.0.0.1:37194 - R:/127.11.184.65:45723] using TOKEN/null
23:01:05.930 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.66:32895
23:01:05.931 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:05.932274 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.267850 13711 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.268183 13711 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.268285 13711 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.276945 13711 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.277125 13711 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.66
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.288197 13711 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.289826 13711 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.291744 13711 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.302176 13717 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.302174 13716 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:06.303085 13719 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.303987 13711 server_base.cc:1061] running on GCE node
23:01:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.304637 13711 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.305721 13711 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.306957 13711 hybrid_clock.cc:648] HybridClock initialized: now 1783465266306885 us; error 58 us; skew 500 ppm
23:01:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.307348 13711 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.309585 13711 webserver.cc:534] Webserver started at http://127.11.184.66:37479/ using document root <none> and password file <none>
23:01:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.310387 13711 fs_manager.cc:362] Metadata directory not provided
23:01:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.310540 13711 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.316164 13711 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
23:01:06.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.319365 13725 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.320914 13711 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
23:01:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.321089 13711 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.321853 13711 fs_report.cc:389] FS layout report
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/data
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:06.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.337693 13711 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:06.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.338657 13711 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:06.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.339003 13711 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:06.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.340065 13711 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:06.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.342305 13732 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:06.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.348582 13711 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:06.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.348703 13711 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:01:06.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.348845 13711 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.352000 13711 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.352125 13711 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
23:01:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.352592 13732 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap starting.
23:01:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.379568 13732 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Log is configured to *not* fsync() on all Append() calls
23:01:06.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.398200 13711 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.66:32895
23:01:06.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.398248 13839 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.66:32895 every 8 connection(s)
23:01:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.400049 13711 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.409296 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 13711
23:01:06.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.424729 13732 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.426406 13732 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap complete.
23:01:06.427 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:06.428 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.65:45723
23:01:06.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:01:06.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.429189 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 13178
23:01:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.429219 13732 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent bootstrapping tablet: real 0.077s	user 0.038s	sys 0.025s
23:01:06.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.442890 13732 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:06.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.445142 13732 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Initialized, Role: FOLLOWER
23:01:06.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.446049 13732 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.448132 13840 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.448550 13840 heartbeater.cc:461] Registering TS with master...
23:01:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.449295 13732 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent starting tablet: real 0.017s	user 0.009s	sys 0.003s
23:01:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.449808 13840 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.453370 12184 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.454985 12184 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.66:53327
23:01:06.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.458287 13840 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.458918 13841 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.459074 13841 heartbeater.cc:461] Registering TS with master...
23:01:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.459765 13841 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.461827 12042 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.462745 13302 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.468349 13844 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.468488 13844 heartbeater.cc:461] Registering TS with master...
23:01:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.468734 13844 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:06.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.470050 12114 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:06.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:06.562055 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 13178
23:01:06.577 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] handling channelInactive
23:01:06.577 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:06.577 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state READY due to: connection closed
23:01:06.578 [DEBUG - Time-limited test] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:06.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.579 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.588 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:06.588 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:06.589 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test)) after lookup
23:01:06.589 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:06.590 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.590 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.591 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.608 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:06.608 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:06.608 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [11ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [12ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [13ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [29ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502))) after lookup
23:01:06.609 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:06.610 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.610 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.610 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.629 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:06.629 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:06.629 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [11ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [12ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [13ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [29ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [31ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [32ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [32ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [50ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415))) after lookup
23:01:06.630 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:06.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.631 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.668 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:06.669 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:06.669 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=91), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [11ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [12ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [13ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [29ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [31ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [32ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [32ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [50ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [51ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [53ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [53ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [87ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [90ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228))) after lookup
23:01:06.669 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:06.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:06.671 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.168 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:07.168 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:07.168 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=590), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [11ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [12ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [13ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [29ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [31ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [32ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [32ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [50ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [51ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [53ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [53ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [87ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [90ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [91ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [93ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [93ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [587ms] refreshing cache from master, [587ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [590ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228) -> (continuation of Deferred@1960630107 after retry RPC@1270454590), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228) -> (continuation of Deferred@1960630107 after retry RPC@1270454590))) after lookup
23:01:07.169 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:07.170 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.170 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.170 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.729 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:07.729 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:07.730 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1152), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [11ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [12ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [13ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [29ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [31ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [32ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [32ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [50ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [51ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [53ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [53ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [87ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [90ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [91ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [93ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [93ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [587ms] refreshing cache from master, [587ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [590ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [591ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [592ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [592ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1147ms] refreshing cache from master, [1147ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1151ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228) -> (continuation of Deferred@1960630107 after retry RPC@1270454590) -> (continuation of Deferred@1858324495 after retry RPC@1765067904), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228) -> (continuation of Deferred@1960630107 after retry RPC@1270454590) -> (continuation of Deferred@1858324495 after retry RPC@1765067904))) after lookup
23:01:07.730 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:07.731 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.732 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.732 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.875211 13854 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:01:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.875470 13854 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.877161 13854 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:07.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:07.878880 13726 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:07.880903 13726 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.884619 13619 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" is_pre_election: true
23:01:07.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.885509 13728 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6
23:01:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.886008 13854 raft_consensus.cc:2749] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
23:01:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.932920 13859 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 82b47e11f28144cd9318488594317dc6)
23:01:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.933140 13859 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.934689 13859 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:07.936667 13550 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:07.938560 13550 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:07.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.943634 13794 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2" is_pre_election: true
23:01:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.944026 13794 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 6.
23:01:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.945053 13553 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2; no voters: 82b47e11f28144cd9318488594317dc6
23:01:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.945500 13859 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Leader pre-election won for term 8
23:01:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.945670 13859 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Starting leader election (detected failure of leader 82b47e11f28144cd9318488594317dc6)
23:01:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.945796 13859 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 7 FOLLOWER]: Advancing to term 8
23:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.948010 13859 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.948695 13859 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 8 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.949358 13794 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2"
23:01:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.949584 13794 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 6 FOLLOWER]: Advancing to term 8
23:01:07.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:07.950225 13550 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.955495 13794 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 8.
23:01:07.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.956038 13553 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2; no voters: 82b47e11f28144cd9318488594317dc6
23:01:07.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.956424 13859 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 FOLLOWER]: Leader election won for term 8
23:01:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.956949 13859 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 LEADER]: Becoming Leader. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Running, Role: LEADER
23:01:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.957532 13859 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:07.961491 12184 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d reported cstate change: term changed from 7 to 8, leader changed from 82b47e11f28144cd9318488594317dc6 (127.11.184.65) to 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67). New cstate: current_term: 8 leader_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: HEALTHY } } }
23:01:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:08.353206 13794 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Refusing update from remote peer 6bc03e9853ba4c04823c3c78a8a98d7d: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
23:01:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:08.354112 13859 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
23:01:08.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:08.363783 13550 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Couldn't send request to peer 82b47e11f28144cd9318488594317dc6. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:01:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:10.678083 13550 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Couldn't send request to peer 82b47e11f28144cd9318488594317dc6. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:01:11.709 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:11.710 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:11.711 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=5132), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [10ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [11ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [12ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [13ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [29ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [31ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [32ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [32ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [50ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [51ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [53ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [53ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [87ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [90ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [91ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [93ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [93ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [587ms] refreshing cache from master, [587ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [590ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [591ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [592ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [592ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1147ms] refreshing cache from master, [1147ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1151ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [1152ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1154ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1154ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [5127ms] refreshing cache from master, [5128ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [5131ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@2023998665(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@3e6ad14b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228) -> (continuation of Deferred@1960630107 after retry RPC@1270454590) -> (continuation of Deferred@1858324495 after retry RPC@1765067904) -> (continuation of Deferred@1731510640 after retry RPC@682316250), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26e517a5 -> wakeup thread Time-limited test -> (continuation of Deferred@1603678417 after retry RPC@1536465502) -> (continuation of Deferred@135719009 after retry RPC@1999619415) -> (continuation of Deferred@1602308414 after retry RPC@1452096228) -> (continuation of Deferred@1960630107 after retry RPC@1270454590) -> (continuation of Deferred@1858324495 after retry RPC@1765067904) -> (continuation of Deferred@1731510640 after retry RPC@682316250))) after lookup
23:01:11.711 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:11.713 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] Successfully connected to peer
23:01:11.732 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x927d1170, L:/127.0.0.1:46916 - R:/127.11.184.67:46593] using TOKEN/null
23:01:11.757 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.65:45723
23:01:11.757 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:01:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:11.758581 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:45723
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40833
23:01:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:01:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.108302 13896 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.108609 13896 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.108693 13896 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.117307 13896 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.117481 13896 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.65
23:01:12.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.128436 13896 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:12.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:12.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:45723
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40833
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:12.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.130793 13896 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:12.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.132936 13896 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.144670 13903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.144893 13906 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.144922 13904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.145603 13896 server_base.cc:1061] running on GCE node
23:01:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.146378 13896 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.147513 13896 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.148813 13896 hybrid_clock.cc:648] HybridClock initialized: now 1783465272148771 us; error 48 us; skew 500 ppm
23:01:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.149147 13896 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.151764 13896 webserver.cc:534] Webserver started at http://127.11.184.65:40833/ using document root <none> and password file <none>
23:01:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.152562 13896 fs_manager.cc:362] Metadata directory not provided
23:01:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.152664 13896 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.158936 13896 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
23:01:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.162179 13912 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:12.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.163914 13896 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.164079 13896 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82b47e11f28144cd9318488594317dc6"
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.164886 13896 fs_report.cc:389] FS layout report
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/data
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:12.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.200937 13896 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:12.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.201932 13896 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:12.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.202196 13896 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:12.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.203264 13896 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.205426 13919 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.212349 13896 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.212479 13896 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:01:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.212641 13896 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:12.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.216275 13896 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:12.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.216377 13896 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
23:01:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.216908 13919 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap starting.
23:01:12.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.245891 13919 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Log is configured to *not* fsync() on all Append() calls
23:01:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.262631 13896 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.65:45723
23:01:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.262735 14026 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.65:45723 every 8 connection(s)
23:01:12.268 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:12.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.264643 13896 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:01:12.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.268168 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 13896
23:01:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.300485 14028 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.301083 14028 heartbeater.cc:461] Registering TS with master...
23:01:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.302546 14028 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:12.302 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:12.304 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.67:46593
23:01:12.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:01:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.304842 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 13535
23:01:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.306214 12043 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:01:12.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.320612 13919 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:12.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.321264 13919 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap complete.
23:01:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.322321 14027 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.322482 14027 heartbeater.cc:461] Registering TS with master...
23:01:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.323148 14027 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.323494 13919 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent bootstrapping tablet: real 0.107s	user 0.036s	sys 0.039s
23:01:12.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.324481 12184 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:01:12.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.326229 12184 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.65:33007
23:01:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.332988 13919 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.334565 13919 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Initialized, Role: FOLLOWER
23:01:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.335811 13919 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:12.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.356222 14033 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:12.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.356433 14033 heartbeater.cc:461] Registering TS with master...
23:01:12.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.356856 14033 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.358429 12113 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:01:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.359400 13659 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.359486 14027 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.361136 13919 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent starting tablet: real 0.038s	user 0.013s	sys 0.018s
23:01:12.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:12.467957 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 13535
23:01:12.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:12.488305 12099 connection.cc:573] server connection from 127.11.184.67:35445 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:01:12.488 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] handling channelInactive
23:01:12.488 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state READY due to: connection closed
23:01:12.489 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: connection closed
23:01:12.508 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:12.508 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:12.509 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [20ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@679399605(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@5aa961e1 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a5c3ac7 -> wakeup thread Time-limited test)) after lookup
23:01:12.509 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:12.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.510 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.511 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.529 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:12.529 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:12.530 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [20ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [21ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [23ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [41ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@679399605(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@5aa961e1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a5c3ac7 -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992))) after lookup
23:01:12.530 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:12.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.532 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.569 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:12.569 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:12.569 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [20ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [21ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [23ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [41ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [42ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [44ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [81ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@679399605(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@5aa961e1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a5c3ac7 -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215))) after lookup
23:01:12.570 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:12.571 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.571 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.572 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.669 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:12.669 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:12.669 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=181), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [20ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [21ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [23ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [41ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [42ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [44ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [81ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [82ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [83ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [84ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [180ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@679399605(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@5aa961e1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215) -> (continuation of Deferred@1838361848 after retry RPC@1598119723), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a5c3ac7 -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215) -> (continuation of Deferred@1838361848 after retry RPC@1598119723))) after lookup
23:01:12.670 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:12.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.671 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.829 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:12.830 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:12.830 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=341), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [20ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [21ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [23ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [41ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [42ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [44ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [81ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [82ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [83ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [84ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [180ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [182ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [183ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [183ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [337ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [341ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@679399605(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@5aa961e1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215) -> (continuation of Deferred@1838361848 after retry RPC@1598119723) -> (continuation of Deferred@1975754916 after retry RPC@548278576), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a5c3ac7 -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215) -> (continuation of Deferred@1838361848 after retry RPC@1598119723) -> (continuation of Deferred@1975754916 after retry RPC@548278576))) after lookup
23:01:12.831 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:12.832 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.832 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:12.832 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593
23:01:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.634344 14041 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:01:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.634624 14041 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.636358 14041 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:13.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:13.637988 13915 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:13.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.638188 14045 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:01:13.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.638381 14045 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:13.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.639415 14045 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:13.641516 13728 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:13.641686 13915 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:13.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:13.643214 13728 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.645728 13794 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2" is_pre_election: true
23:01:13.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.646086 13794 raft_consensus.cc:2393] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82b47e11f28144cd9318488594317dc6 in current term 8: Already voted for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in this term.
23:01:13.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.646884 13916 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82b47e11f28144cd9318488594317dc6; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2
23:01:13.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.647353 14041 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 7 FOLLOWER]: Advancing to term 8
23:01:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.650120 13981 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6" is_pre_election: true
23:01:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.651854 14041 raft_consensus.cc:2749] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 8 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
23:01:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.652164 13981 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 8.
23:01:13.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.653620 13726 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d
23:01:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.653900 14045 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Leader pre-election won for term 9
23:01:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.654034 14045 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 6bc03e9853ba4c04823c3c78a8a98d7d)
23:01:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.654137 14045 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 8 FOLLOWER]: Advancing to term 9
23:01:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.656255 14045 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:13.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.656886 14045 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 9 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:13.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.657359 13981 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "fa076f9a47194d2daca748165b9575e2" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "82b47e11f28144cd9318488594317dc6"
23:01:13.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.657554 13981 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 8 FOLLOWER]: Advancing to term 9
23:01:13.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:13.658375 13728 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111)
23:01:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.659982 13981 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa076f9a47194d2daca748165b9575e2 in term 9.
23:01:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.660363 13726 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 82b47e11f28144cd9318488594317dc6, fa076f9a47194d2daca748165b9575e2; no voters: 6bc03e9853ba4c04823c3c78a8a98d7d
23:01:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.660614 14045 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 FOLLOWER]: Leader election won for term 9
23:01:13.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.661016 14045 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 LEADER]: Becoming Leader. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Running, Role: LEADER
23:01:13.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.661494 14045 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:13.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:13.665256 12184 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 reported cstate change: term changed from 8 to 9, leader changed from 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67) to fa076f9a47194d2daca748165b9575e2 (127.11.184.66). New cstate: current_term: 9 leader_uuid: "fa076f9a47194d2daca748165b9575e2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:01:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.089454 13981 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Refusing update from remote peer fa076f9a47194d2daca748165b9575e2: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
23:01:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.090340 14045 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
23:01:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.104821 13728 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 -> Peer 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593): Couldn't send request to peer 6bc03e9853ba4c04823c3c78a8a98d7d. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.67:46593: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:01:14.189 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:14.189 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:14.190 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1701), Traces: [0ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [1ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [20ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [21ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [23ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [41ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [42ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [44ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [81ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [82ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [83ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [84ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [180ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [182ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [183ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [183ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [337ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [341ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [342ms] sending RPC to server 6bc03e9853ba4c04823c3c78a8a98d7d, [344ms] received response from server 6bc03e9853ba4c04823c3c78a8a98d7d: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [344ms] delaying RPC due to: Network error: Failed to connect to peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593): Connection refused: /127.11.184.67:46593, [1697ms] refreshing cache from master, [1697ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1700ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@679399605(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@5aa961e1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215) -> (continuation of Deferred@1838361848 after retry RPC@1598119723) -> (continuation of Deferred@1975754916 after retry RPC@548278576) -> (continuation of Deferred@730281970 after retry RPC@2070975523), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a5c3ac7 -> wakeup thread Time-limited test -> (continuation of Deferred@1237319338 after retry RPC@1111387992) -> (continuation of Deferred@368671406 after retry RPC@760473215) -> (continuation of Deferred@1838361848 after retry RPC@1598119723) -> (continuation of Deferred@1975754916 after retry RPC@548278576) -> (continuation of Deferred@730281970 after retry RPC@2070975523))) after lookup
23:01:14.191 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:14.192 [DEBUG - kudu-nio-6] (Connection.java:757) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] Successfully connected to peer
23:01:14.219 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xc26743ac, L:/127.0.0.1:37706 - R:/127.11.184.66:32895] using TOKEN/null
23:01:14.254 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.67:46593
23:01:14.254 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:01:14.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.255584 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:46593
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41735
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.595598 14073 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.595968 14073 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.596094 14073 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.605075 14073 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.605316 14073 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.67
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.616267 14073 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.67:46593
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.67
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41735
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.67
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/logs
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.618367 14073 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.620311 14073 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.629886 14081 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.629912 14080 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:14.630800 14083 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:14.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.631850 14073 server_base.cc:1061] running on GCE node
23:01:14.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.632608 14073 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:14.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.633631 14073 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.634879 14073 hybrid_clock.cc:648] HybridClock initialized: now 1783465274634817 us; error 45 us; skew 500 ppm
23:01:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.635295 14073 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:14.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.637488 14073 webserver.cc:534] Webserver started at http://127.11.184.67:41735/ using document root <none> and password file <none>
23:01:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.638268 14073 fs_manager.cc:362] Metadata directory not provided
23:01:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.638404 14073 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:14.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.644400 14073 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
23:01:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.647360 14089 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.648703 14073 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
23:01:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.648891 14073 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:01:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:41 on dist-test-slave-1pcn"
23:01:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.649636 14073 fs_report.cc:389] FS layout report
23:01:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/wal
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/data
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.668459 14073 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.669410 14073 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.669763 14073 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:14.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.670853 14073 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.673130 14096 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.679090 14073 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.679273 14073 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:01:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.679414 14073 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:14.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.682422 14073 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:14.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.682550 14073 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
23:01:14.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.682906 14096 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap starting.
23:01:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.710376 14096 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Log is configured to *not* fsync() on all Append() calls
23:01:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.726043 14073 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.67:46593
23:01:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.726125 14203 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.67:46593 every 8 connection(s)
23:01:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.727573 14073 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-2/data/info.pb
23:01:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.734251 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 14073
23:01:14.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.755543 14096 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.756420 14096 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Bootstrap complete.
23:01:14.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.758527 14096 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent bootstrapping tablet: real 0.076s	user 0.059s	sys 0.013s
23:01:14.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.766129 14096 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:14.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.767354 14096 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Initialized, Role: FOLLOWER
23:01:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.768240 14096 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.780511 14096 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d: Time spent starting tablet: real 0.022s	user 0.019s	sys 0.000s
23:01:14.789 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:14.790 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.66:32895
23:01:14.790 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:01:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.790844 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 13711
23:01:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.797585 14204 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.798070 14204 heartbeater.cc:461] Registering TS with master...
23:01:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.799329 14204 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.801090 14205 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.801270 14205 heartbeater.cc:461] Registering TS with master...
23:01:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.801820 14205 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.802557 12184 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:14.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.803272 12042 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.804121 12184 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.67:48333
23:01:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.806192 14206 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.806347 14206 heartbeater.cc:461] Registering TS with master...
23:01:14.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.806871 14206 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:14.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.808501 12113 ts_manager.cc:194] Re-registered known tserver with Master: 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.810364 14204 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:14.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.813452 13835 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:14.912158 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 13711
23:01:14.931 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:14.932 [DEBUG - kudu-nio-6] (Connection.java:239) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] handling channelInactive
23:01:14.932 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state READY due to: connection closed
23:01:14.946 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:14.947 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.947 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.947 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.968 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:14.968 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:14.969 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=3, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [36ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1033020906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@7b2049 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@30724fbc -> wakeup thread Time-limited test)) after lookup
23:01:14.969 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:14.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.971 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.988 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:14.989 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:14.989 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=5, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [36ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [37ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [39ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [39ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [56ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1033020906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@7b2049 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@30724fbc -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299))) after lookup
23:01:14.989 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:14.990 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:14.991 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.028 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:15.028 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:15.029 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=7, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=97), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [36ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [37ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [39ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [39ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [56ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [57ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [59ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [59ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [96ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1033020906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@7b2049 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@30724fbc -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871))) after lookup
23:01:15.029 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:15.030 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.030 [DEBUG - kudu-nio-2] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.030 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.228 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:15.229 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:15.229 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=9, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=297), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [36ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [37ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [39ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [39ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [56ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [57ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [59ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [59ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [96ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [97ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [98ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [98ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [293ms] refreshing cache from master, [293ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [296ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1033020906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@7b2049 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871) -> (continuation of Deferred@1808868864 after retry RPC@362944186), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@30724fbc -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871) -> (continuation of Deferred@1808868864 after retry RPC@362944186))) after lookup
23:01:15.229 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:15.230 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.230 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.230 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.909 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:15.909 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)[L]]}]
23:01:15.910 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=11, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=977), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [36ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [37ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [39ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [39ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [56ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [57ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [59ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [59ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [96ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [97ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [98ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [98ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [293ms] refreshing cache from master, [293ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [296ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [297ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [298ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [298ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [973ms] refreshing cache from master, [974ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [976ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1033020906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@7b2049 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871) -> (continuation of Deferred@1808868864 after retry RPC@362944186) -> (continuation of Deferred@1316674591 after retry RPC@1265112236), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@30724fbc -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871) -> (continuation of Deferred@1808868864 after retry RPC@362944186) -> (continuation of Deferred@1316674591 after retry RPC@1265112236))) after lookup
23:01:15.910 [DEBUG - kudu-nio-2] (Connection.java:748) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] connecting to peer
23:01:15.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] cleaning up while in state CONNECTING due to: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:15.912 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895
23:01:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.229602 14219 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:01:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.229830 14219 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.230701 14219 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:16.234102 13916 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:01:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.240614 14158 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" is_pre_election: true
23:01:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.241039 14158 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82b47e11f28144cd9318488594317dc6 in term 8.
23:01:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.241945 13915 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:01:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.242295 14219 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Leader pre-election won for term 10
23:01:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.242396 14219 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Starting leader election (detected failure of leader fa076f9a47194d2daca748165b9575e2)
23:01:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.242456 14219 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 9 FOLLOWER]: Advancing to term 10
23:01:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.244514 14219 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.245105 14219 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 10 election: Requested vote from peers fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895), 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67:46593)
23:01:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.245728 14158 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "82b47e11f28144cd9318488594317dc6" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d"
23:01:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.245934 14158 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 8 FOLLOWER]: Advancing to term 10
23:01:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:16.246659 13916 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111)
23:01:16.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.249778 14158 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82b47e11f28144cd9318488594317dc6 in term 10.
23:01:16.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.250198 13915 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, 82b47e11f28144cd9318488594317dc6; no voters: fa076f9a47194d2daca748165b9575e2
23:01:16.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.250489 14219 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 10 FOLLOWER]: Leader election won for term 10
23:01:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.250916 14219 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 10 LEADER]: Becoming Leader. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Running, Role: LEADER
23:01:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.251473 14219 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.255259 12184 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 reported cstate change: term changed from 9 to 10, leader changed from fa076f9a47194d2daca748165b9575e2 (127.11.184.66) to 82b47e11f28144cd9318488594317dc6 (127.11.184.65). New cstate: current_term: 10 leader_uuid: "82b47e11f28144cd9318488594317dc6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: UNKNOWN } } }
23:01:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.659369 14158 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Refusing update from remote peer 82b47e11f28144cd9318488594317dc6: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
23:01:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:16.660329 14219 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
23:01:16.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:16.676844 13916 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 -> Peer fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895): Couldn't send request to peer fa076f9a47194d2daca748165b9575e2. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.66:32895: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:01:18.028 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:18.029 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:18.029 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=13, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3097), Traces: [0ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [15ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [15ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [36ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [37ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [39ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [39ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [56ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [57ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [59ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [59ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [96ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [97ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [98ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [98ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [293ms] refreshing cache from master, [293ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [296ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [297ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [298ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [298ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [973ms] refreshing cache from master, [974ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [976ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [978ms] sending RPC to server fa076f9a47194d2daca748165b9575e2, [980ms] received response from server fa076f9a47194d2daca748165b9575e2: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [980ms] delaying RPC due to: Network error: Failed to connect to peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895): Connection refused: /127.11.184.66:32895, [3093ms] refreshing cache from master, [3093ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [3096ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@1033020906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@7b2049 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871) -> (continuation of Deferred@1808868864 after retry RPC@362944186) -> (continuation of Deferred@1316674591 after retry RPC@1265112236) -> (continuation of Deferred@1557573852 after retry RPC@1469059742), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@30724fbc -> wakeup thread Time-limited test -> (continuation of Deferred@201656640 after retry RPC@782809299) -> (continuation of Deferred@1384852983 after retry RPC@1790016871) -> (continuation of Deferred@1808868864 after retry RPC@362944186) -> (continuation of Deferred@1316674591 after retry RPC@1265112236) -> (continuation of Deferred@1557573852 after retry RPC@1469059742))) after lookup
23:01:18.029 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:18.030 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] Successfully connected to peer
23:01:18.053 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x871f5b30, L:/127.0.0.1:45680 - R:/127.11.184.65:45723] using TOKEN/null
23:01:18.076 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.66:32895
23:01:18.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.076988 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.420009 14257 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.420339 14257 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.420423 14257 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.428928 14257 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.429112 14257 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.66
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.439769 14257 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.66:32895
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.66
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37479
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.66
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/logs
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:18.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.441414 14257 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.443478 14257 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:18.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.453819 14262 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:18.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.454092 14265 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:18.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.453845 14263 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:18.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.454806 14257 server_base.cc:1061] running on GCE node
23:01:18.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.455497 14257 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.456388 14257 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.457608 14257 hybrid_clock.cc:648] HybridClock initialized: now 1783465278457557 us; error 48 us; skew 500 ppm
23:01:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.457906 14257 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:18.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.460441 14257 webserver.cc:534] Webserver started at http://127.11.184.66:37479/ using document root <none> and password file <none>
23:01:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.461166 14257 fs_manager.cc:362] Metadata directory not provided
23:01:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.461261 14257 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.466925 14257 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.001s
23:01:18.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.469973 14271 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.471642 14257 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.471786 14257 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa076f9a47194d2daca748165b9575e2"
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.472599 14257 fs_report.cc:389] FS layout report
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/wal
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/data
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.495790 14257 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.496652 14257 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.497005 14257 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:18.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.498123 14257 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:18.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.500361 14280 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.506490 14257 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.506647 14257 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
23:01:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.506804 14257 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.509992 14257 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.510124 14257 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
23:01:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.510783 14280 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap starting.
23:01:18.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.541833 14280 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Log is configured to *not* fsync() on all Append() calls
23:01:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.561322 14257 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.66:32895
23:01:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.561364 14387 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.66:32895 every 8 connection(s)
23:01:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.562832 14257 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-1/data/info.pb
23:01:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.565030 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 14257
23:01:18.565 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:18.591 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:18.599 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.11.184.65:45723
23:01:18.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:01:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.600123 14280 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.600210 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 13896
23:01:18.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.606212 14280 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Bootstrap complete.
23:01:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.609915 14280 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent bootstrapping tablet: real 0.100s	user 0.054s	sys 0.024s
23:01:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.610064 14388 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.610518 14388 heartbeater.cc:461] Registering TS with master...
23:01:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.611720 14388 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.615197 12185 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:18.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.615980 14280 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:18.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.616847 14280 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa076f9a47194d2daca748165b9575e2, State: Initialized, Role: FOLLOWER
23:01:18.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.616958 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.66:56747
23:01:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.617803 14389 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.617987 14389 heartbeater.cc:461] Registering TS with master...
23:01:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.617637 14280 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.618553 14389 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.619755 14390 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.619905 14390 heartbeater.cc:461] Registering TS with master...
23:01:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.620314 14390 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.620469 14022 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.621937 12113 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.621950 12042 ts_manager.cc:194] Re-registered known tserver with Master: fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.622100 14388 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:18.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.622892 14280 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2: Time spent starting tablet: real 0.013s	user 0.008s	sys 0.001s
23:01:18.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:18.721338 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 13896
23:01:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.740005 12101 connection.cc:573] server connection from 127.11.184.65:51231 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:01:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.740032 12029 connection.cc:573] server connection from 127.11.184.65:41915 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:01:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.740134 14093 connection.cc:573] server connection from 127.11.184.65:42093 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:01:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:18.740752 12169 connection.cc:573] server connection from 127.11.184.65:33007 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:01:18.740 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] handling channelInactive
23:01:18.741 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state READY due to: connection closed
23:01:18.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:18.742 [DEBUG - Time-limited test] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:18.743 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.743 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.743 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.747 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:18.748 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:18.748 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=6), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test)) after lookup
23:01:18.748 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:18.749 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.749 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.750 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.768 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:18.768 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:18.769 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [7ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [9ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [9ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [27ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366))) after lookup
23:01:18.769 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:18.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.770 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.808 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:18.808 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:18.809 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [7ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [9ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [9ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [27ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [28ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [30ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [67ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267))) after lookup
23:01:18.809 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:18.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.810 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.908 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:18.909 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:18.909 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=168), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [7ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [9ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [9ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [27ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [28ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [30ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [67ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [68ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [69ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [69ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413))) after lookup
23:01:18.910 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:18.911 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.911 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:18.911 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:19.428 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:19.428 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:19.429 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=687), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [7ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [9ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [9ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [27ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [28ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [30ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [67ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [68ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [69ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [69ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [168ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [170ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [170ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [684ms] refreshing cache from master, [684ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [687ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413) -> (continuation of Deferred@258785655 after retry RPC@151675637), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413) -> (continuation of Deferred@258785655 after retry RPC@151675637))) after lookup
23:01:19.429 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:19.430 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:19.430 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:19.430 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:19.748 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:19.748 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593),82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)[L],fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:19.749 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1007), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [7ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [9ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [9ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [27ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [28ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [30ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [67ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [68ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [69ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [69ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [168ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [170ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [170ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [684ms] refreshing cache from master, [684ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [687ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [688ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [689ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [690ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1004ms] refreshing cache from master, [1004ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1007ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413) -> (continuation of Deferred@258785655 after retry RPC@151675637) -> (continuation of Deferred@1844419911 after retry RPC@933779397), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413) -> (continuation of Deferred@258785655 after retry RPC@151675637) -> (continuation of Deferred@1844419911 after retry RPC@933779397))) after lookup
23:01:19.749 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] connecting to peer
23:01:19.750 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:19.750 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] cleaning up while in state CONNECTING due to: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:19.750 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723) for tablet d3889c79e97e44dfb205d9875fe68a97: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723
23:01:20.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.222375 14403 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 82b47e11f28144cd9318488594317dc6)
23:01:20.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.222674 14403 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:20.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.224452 14403 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:20.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:20.225821 14090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:20.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:20.227660 14090 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.234086 14342 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2" is_pre_election: true
23:01:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.234608 14342 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 9.
23:01:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.235679 14093 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2; no voters: 82b47e11f28144cd9318488594317dc6
23:01:20.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.236135 14403 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Leader pre-election won for term 11
23:01:20.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.236275 14403 raft_consensus.cc:493] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Starting leader election (detected failure of leader 82b47e11f28144cd9318488594317dc6)
23:01:20.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.236368 14403 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 10 FOLLOWER]: Advancing to term 11
23:01:20.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.238972 14403 raft_consensus.cc:515] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:20.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.239629 14403 leader_election.cc:290] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 11 election: Requested vote from peers 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723), fa076f9a47194d2daca748165b9575e2 (127.11.184.66:32895)
23:01:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.240532 14342 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d3889c79e97e44dfb205d9875fe68a97" candidate_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "fa076f9a47194d2daca748165b9575e2"
23:01:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.240783 14342 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 9 FOLLOWER]: Advancing to term 11
23:01:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:20.241396 14090 leader_election.cc:336] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111)
23:01:20.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.247548 14342 raft_consensus.cc:2468] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bc03e9853ba4c04823c3c78a8a98d7d in term 11.
23:01:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.248168 14093 leader_election.cc:304] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6bc03e9853ba4c04823c3c78a8a98d7d, fa076f9a47194d2daca748165b9575e2; no voters: 82b47e11f28144cd9318488594317dc6
23:01:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.248523 14403 raft_consensus.cc:2804] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 11 FOLLOWER]: Leader election won for term 11
23:01:20.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.249027 14403 raft_consensus.cc:697] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [term 11 LEADER]: Becoming Leader. State: Replica: 6bc03e9853ba4c04823c3c78a8a98d7d, State: Running, Role: LEADER
23:01:20.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.249559 14403 consensus_queue.cc:237] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:20.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.255256 12185 catalog_manager.cc:5696] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d reported cstate change: term changed from 10 to 11, leader changed from 82b47e11f28144cd9318488594317dc6 (127.11.184.65) to 6bc03e9853ba4c04823c3c78a8a98d7d (127.11.184.67). New cstate: current_term: 11 leader_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } health_report { overall_health: HEALTHY } } }
23:01:20.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:20.772307 14090 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Couldn't send request to peer 82b47e11f28144cd9318488594317dc6. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:01:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.775215 14342 raft_consensus.cc:1275] T d3889c79e97e44dfb205d9875fe68a97 P fa076f9a47194d2daca748165b9575e2 [term 11 FOLLOWER]: Refusing update from remote peer 6bc03e9853ba4c04823c3c78a8a98d7d: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
23:01:20.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:20.776172 14420 consensus_queue.cc:1048] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
23:01:23.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:23.270949 14090 consensus_peers.cc:597] T d3889c79e97e44dfb205d9875fe68a97 P 6bc03e9853ba4c04823c3c78a8a98d7d -> Peer 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723): Couldn't send request to peer 82b47e11f28144cd9318488594317dc6. Status: Network error: Client connection negotiation failed: client connection to 127.11.184.65:45723: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:01:23.708 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d3889c79e97e44dfb205d9875fe68a97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849' with partition [<start>, <end>)
23:01:23.709 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d3889c79e97e44dfb205d9875fe68a97@[6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)[L],82b47e11f28144cd9318488594317dc6(127.11.184.65:45723),fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)]}]
23:01:23.709 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d3889c79e97e44dfb205d9875fe68a97, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4967), Traces: [0ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [2ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [2ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [6ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [7ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [9ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [9ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [27ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [28ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [29ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [30ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [67ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [68ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [69ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [69ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [167ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [168ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [170ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [170ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [684ms] refreshing cache from master, [684ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [687ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [688ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [689ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [690ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1004ms] refreshing cache from master, [1004ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [1007ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, [1008ms] sending RPC to server 82b47e11f28144cd9318488594317dc6, [1009ms] received response from server 82b47e11f28144cd9318488594317dc6: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [1009ms] delaying RPC due to: Network error: Failed to connect to peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723): Connection refused: /127.11.184.65:45723, [4964ms] refreshing cache from master, [4964ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.184.124:42425, [4967ms] Sub RPC GetTableLocations: received response from server master-127.11.184.124:42425: OK, deferred=Deferred@458527409(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f53b836bed0@4c8767a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413) -> (continuation of Deferred@258785655 after retry RPC@151675637) -> (continuation of Deferred@1844419911 after retry RPC@933779397) -> (continuation of Deferred@260337150 after retry RPC@2053576116), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab24a7e -> wakeup thread Time-limited test -> (continuation of Deferred@2081514744 after retry RPC@1153970366) -> (continuation of Deferred@2060911895 after retry RPC@2141161267) -> (continuation of Deferred@1453462941 after retry RPC@1434957413) -> (continuation of Deferred@258785655 after retry RPC@151675637) -> (continuation of Deferred@1844419911 after retry RPC@933779397) -> (continuation of Deferred@260337150 after retry RPC@2053576116))) after lookup
23:01:23.710 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] connecting to peer
23:01:23.711 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] Successfully connected to peer
23:01:23.729 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x426321c5, L:/127.0.0.1:47788 - R:/127.11.184.67:46593] using TOKEN/null
23:01:23.753 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.11.184.65:45723
23:01:23.753 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:23.754164 12001 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:01:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:45723
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40833
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:01:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.102654 14445 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:01:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.102974 14445 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:01:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.103073 14445 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:01:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.111521 14445 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:01:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.111719 14445 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.184.65
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.121711 14445 tablet_server_runner.cc:78] Tablet server non-default flags:
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.184.84:43957
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.184.65:45723
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.184.65
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40833
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.184.126:46381,127.11.184.125:41661,127.11.184.124:42425
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:01:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.184.65
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/logs
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on bdcb31816ec0
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16828
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:01:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.123505 14445 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:01:24.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.125337 14445 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:01:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.135529 14453 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.136124 14455 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:01:24.136412 14452 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:01:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.136682 14445 server_base.cc:1061] running on GCE node
23:01:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.137238 14445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:01:24.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.138164 14445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:01:24.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.139366 14445 hybrid_clock.cc:648] HybridClock initialized: now 1783465284139311 us; error 44 us; skew 500 ppm
23:01:24.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.139690 14445 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:01:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.142067 14445 webserver.cc:534] Webserver started at http://127.11.184.65:40833/ using document root <none> and password file <none>
23:01:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.142782 14445 fs_manager.cc:362] Metadata directory not provided
23:01:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.142874 14445 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:01:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.148653 14445 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.004s
23:01:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.151614 14461 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:01:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.153136 14445 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.002s
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.153276 14445 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data,/tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82b47e11f28144cd9318488594317dc6"
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:00:40 on dist-test-slave-1pcn"
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.153960 14445 fs_report.cc:389] FS layout report
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/wal
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/data
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:01:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:01:24.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.179682 14445 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:01:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.180588 14445 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:01:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.180859 14445 kserver.cc:163] Server-wide thread pool size limit: 3276
23:01:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.181836 14445 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:01:24.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.184150 14468 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:01:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.190198 14445 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:01:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.190296 14445 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
23:01:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.190402 14445 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:01:24.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.193771 14445 ts_tablet_manager.cc:616] Registered 1 tablets
23:01:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.193871 14445 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
23:01:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.194420 14468 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap starting.
23:01:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.225274 14468 log.cc:826] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Log is configured to *not* fsync() on all Append() calls
23:01:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.246234 14445 rpc_server.cc:307] RPC server started. Bound to: 127.11.184.65:45723
23:01:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.246248 14575 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.184.65:45723 every 8 connection(s)
23:01:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.248373 14445 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjzpEJp/test-tmp/mini-kudu-cluster150263414297899824/ts-0/data/info.pb
23:01:24.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.252890 12001 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu as pid 14445
23:01:24.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:01:24.278 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:24.287 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465241849, tablet=d3889c79e97e44dfb205d9875fe68a97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:01:24.295 [DEBUG - main] (Connection.java:575) [peer 82b47e11f28144cd9318488594317dc6(127.11.184.65:45723)] disconnecting while in state TERMINATED
23:01:24.296 [DEBUG - main] (Connection.java:575) [peer master-127.11.184.125:41661(127.11.184.125:41661)] disconnecting while in state READY
23:01:24.296 [DEBUG - main] (Connection.java:575) [peer master-127.11.184.126:46381(127.11.184.126:46381)] disconnecting while in state READY
23:01:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.292800 14576 heartbeater.cc:344] Connected to a master server at 127.11.184.124:42425
23:01:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.293188 14576 heartbeater.cc:461] Registering TS with master...
23:01:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.294708 14576 heartbeater.cc:507] Master 127.11.184.124:42425 requested a full tablet report, sending...
23:01:24.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.297952 14577 heartbeater.cc:344] Connected to a master server at 127.11.184.126:46381
23:01:24.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.298101 14577 heartbeater.cc:461] Registering TS with master...
23:01:24.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.298702 14577 heartbeater.cc:507] Master 127.11.184.126:46381 requested a full tablet report, sending...
23:01:24.300 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.184.126:46381(127.11.184.126:46381)] handling channelInactive
23:01:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.301362 12185 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:01:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.301517 12042 ts_manager.cc:194] Re-registered known tserver with Master: 82b47e11f28144cd9318488594317dc6 (127.11.184.65:45723)
23:01:24.303 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.11.184.125:41661(127.11.184.125:41661)] handling channelInactive
23:01:24.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.302953 12185 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.184.65:49965
23:01:24.303 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.184.125:41661(127.11.184.125:41661)] cleaning up while in state READY due to: connection closed
23:01:24.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.304837 14468 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:01:24.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.184.126:46381(127.11.184.126:46381)] cleaning up while in state READY due to: connection closed
23:01:24.305 [DEBUG - main] (Connection.java:575) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] disconnecting while in state READY
23:01:24.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.305569 14468 tablet_bootstrap.cc:492] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Bootstrap complete.
23:01:24.305 [DEBUG - main] (Connection.java:575) [peer master-127.11.184.124:42425(127.11.184.124:42425)] disconnecting while in state READY
23:01:24.305 [DEBUG - main] (Connection.java:575) [peer fa076f9a47194d2daca748165b9575e2(127.11.184.66:32895)] disconnecting while in state TERMINATED
23:01:24.306 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] handling channelInactive
23:01:24.306 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6bc03e9853ba4c04823c3c78a8a98d7d(127.11.184.67:46593)] cleaning up while in state READY due to: connection closed
23:01:24.307 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:01:24.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.307304 14468 ts_tablet_manager.cc:1403] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent bootstrapping tablet: real 0.113s	user 0.075s	sys 0.015s
23:01:24.310 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.11.184.124:42425(127.11.184.124:42425)] handling channelInactive
23:01:24.310 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.184.124:42425(127.11.184.124:42425)] cleaning up while in state READY due to: connection closed
23:01:24.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.312355 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 14445
23:01:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.312830 14468 raft_consensus.cc:359] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.313864 14468 raft_consensus.cc:740] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82b47e11f28144cd9318488594317dc6, State: Initialized, Role: FOLLOWER
23:01:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.314621 14468 consensus_queue.cc:260] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82b47e11f28144cd9318488594317dc6" member_type: VOTER last_known_addr { host: "127.11.184.65" port: 45723 } } peers { permanent_uuid: "fa076f9a47194d2daca748165b9575e2" member_type: VOTER last_known_addr { host: "127.11.184.66" port: 32895 } } peers { permanent_uuid: "6bc03e9853ba4c04823c3c78a8a98d7d" member_type: VOTER last_known_addr { host: "127.11.184.67" port: 46593 } }
23:01:24.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.316954 14576 heartbeater.cc:499] Master 127.11.184.124:42425 was elected leader, sending a full tablet report...
23:01:24.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.317672 14468 ts_tablet_manager.cc:1434] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6: Time spent starting tablet: real 0.010s	user 0.009s	sys 0.000s
23:01:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.335723 14530 raft_consensus.cc:3060] T d3889c79e97e44dfb205d9875fe68a97 P 82b47e11f28144cd9318488594317dc6 [term 10 FOLLOWER]: Advancing to term 11
23:01:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.338018 14578 heartbeater.cc:344] Connected to a master server at 127.11.184.125:41661
23:01:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.338215 14578 heartbeater.cc:461] Registering TS with master...
23:01:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.338523 14571 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.338647 14578 heartbeater.cc:507] Master 127.11.184.125:41661 requested a full tablet report, sending...
23:01:24.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.451623 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 14445
23:01:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.470085 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 14257
23:01:24.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.479346 14383 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.571512 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 14257
23:01:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.589797 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 14073
23:01:24.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.598697 14199 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.719314 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 14073
23:01:24.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.739045 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12012
23:01:24.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.740268 12073 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:24.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.856827 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12012
23:01:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.879458 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12080
23:01:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.880715 12144 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:24.998929 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12080
23:01:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:25.022418 12001 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu pid 12151
23:01:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:25.023627 12215 generic_service.cc:196] Checking for leaks (request via RPC)
23:01:25.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:01:25.145179 12001 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskjzpEJp/build/asan/bin/kudu with pid 12151
23:01:25.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:01:25Z chronyd exiting

Time: 48.587

OK (1 test)

