JUnit version 4.13.2
.12:08:16.782 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783512496773
12:08:16.837 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
12:08:16.875 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
12:08:16.890 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDGNhPW/test-tmp
12:08:16.913 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
12:08:16.916 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDGNhPW/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
12:08:17.092 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

12:08:17.110 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:08:17.111 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

12:08:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.113312  3327 mini_oidc.cc:150] Starting JWKS server
12:08:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.115473  3327 webserver.cc:533] Webserver started at http://127.0.0.1:38167/ using document root <none> and password file <none>
12:08:17.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.115665  3327 mini_oidc.cc:186] Starting OIDC Discovery server
12:08:17.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.115950  3327 webserver.cc:533] Webserver started at http://127.0.0.1:35807/ using document root <none> and password file <none>
12:08:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:08:17Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
12:08:17.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:08:17Z Disabled control of system clock
12:08:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.132678  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal
12:08:17.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data
12:08:17.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/logs
12:08:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data/info.pb
12:08:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:08:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:08:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:08:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.254:33189
12:08:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.254
12:08:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:17.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:08:17.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:08:17.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json with env {}
12:08:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.234437  3338 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.234669  3338 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.234731  3338 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:17.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.238595  3338 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:08:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.238657  3338 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.238685  3338 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:08:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.238709  3338 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:08:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.238734  3338 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:08:17.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.238754  3338 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38167/jwks.json
12:08:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.243007  3338 master_runner.cc:387] Master server non-default flags:
12:08:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data
12:08:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal
12:08:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:08:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:08:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.254:33189
12:08:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:08:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:08:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
12:08:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data/info.pb
12:08:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.254
12:08:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3338
12:08:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/logs
12:08:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:08:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:17.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:17.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:17.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:17.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.244484  3338 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:17.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.245954  3338 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:17.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.252494  3343 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.252594  3338 server_base.cc:1061] running on GCE node
12:08:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.252493  3344 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.252494  3346 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.253410  3338 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:17.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.254496  3338 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.255671  3338 hybrid_clock.cc:648] HybridClock initialized: now 1783512497255620 us; error 65 us; skew 500 ppm
12:08:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.257623  3338 webserver.cc:533] Webserver started at http://127.3.63.254:36573/ using document root <none> and password file <none>
12:08:17.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.258247  3338 fs_manager.cc:362] Metadata directory not provided
12:08:17.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.258309  3338 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:17.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.258534  3338 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:08:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.260247  3338 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data/instance:
12:08:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4623f53aa4ed471b852d610ec0c683b4"
12:08:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.260756  3338 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal/instance:
12:08:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4623f53aa4ed471b852d610ec0c683b4"
12:08:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.264245  3338 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.004s	sys 0.001s
12:08:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.266402  3352 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:17.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.267489  3338 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.267586  3338 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4623f53aa4ed471b852d610ec0c683b4"
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.267656  3338 fs_report.cc:389] FS layout report
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal
12:08:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data/data
12:08:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.284696  3338 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.285323  3338 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.285537  3338 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:17.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.292280  3338 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.254:33189
12:08:17.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.292303  3404 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.254:33189 every 8 connection(s)
12:08:17.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.293634  3338 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/data/info.pb
12:08:17.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.296376  3405 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:08:17.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.298717  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 3338
12:08:17.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.298864  3327 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-0/wal/instance
12:08:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.299480  3405 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.300542  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/logs
12:08:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data/info.pb
12:08:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.253:39047
12:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.253
12:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:08:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:08:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json with env {}
12:08:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.315557  3405 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.316437  3354 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.253:39047: connect: Connection refused (error 111)
12:08:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.317516  3405 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.253:39047: Network error: Client connection negotiation failed: client connection to 127.3.63.253:39047: connect: Connection refused (error 111)
12:08:17.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.366724  3405 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } attempt: 1
12:08:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.368284  3405 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.253:39047: Network error: Client connection negotiation failed: client connection to 127.3.63.253:39047: connect: Connection refused (error 111)
12:08:17.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.402253  3407 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.402499  3407 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.402562  3407 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.406603  3407 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:08:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.406675  3407 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.406704  3407 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:08:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.406728  3407 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:08:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.406754  3407 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:08:17.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.406771  3407 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38167/jwks.json
12:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.411178  3407 master_runner.cc:387] Master server non-default flags:
12:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.253:39047
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
12:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data/info.pb
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.253
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3407
12:08:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/logs
12:08:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:08:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.412667  3407 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.414129  3407 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.420475  3418 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.420475  3416 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.420475  3415 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.421128  3407 server_base.cc:1061] running on GCE node
12:08:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.422610  3407 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:17.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.423723  3407 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:17.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.424985  3407 hybrid_clock.cc:648] HybridClock initialized: now 1783512497424932 us; error 26 us; skew 500 ppm
12:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.427314  3407 webserver.cc:533] Webserver started at http://127.3.63.253:35451/ using document root <none> and password file <none>
12:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.428396  3407 fs_manager.cc:362] Metadata directory not provided
12:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.428480  3407 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.428697  3407 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:08:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.431128  3407 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data/instance:
12:08:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2a52f918a5a4cddb9667c7584fcb515"
12:08:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.431680  3407 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal/instance:
12:08:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2a52f918a5a4cddb9667c7584fcb515"
12:08:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.435890  3407 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.003s	sys 0.001s
12:08:17.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.436439  3405 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } attempt: 2
12:08:17.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.438107  3405 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.253:39047: Network error: Client connection negotiation failed: client connection to 127.3.63.253:39047: connect: Connection refused (error 111)
12:08:17.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.438994  3424 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.440179  3407 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
12:08:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.440286  3407 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal
12:08:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2a52f918a5a4cddb9667c7584fcb515"
12:08:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.440377  3407 fs_report.cc:389] FS layout report
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data/data
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.472558  3407 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.473265  3407 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.473413  3407 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.480510  3407 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.253:39047
12:08:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.480526  3476 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.253:39047 every 8 connection(s)
12:08:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.481472  3407 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/data/info.pb
12:08:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.484154  3477 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:08:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.486982  3477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.487463  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 3407
12:08:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.487555  3327 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-1/wal/instance
12:08:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.488219  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal
12:08:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data
12:08:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/logs
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data/info.pb
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:08:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.252:36877
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.252
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:08:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json with env {}
12:08:17.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.499403  3477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.508419  3477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.509269  3426 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.510289  3477 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.252:36877: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.529287  3405 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } attempt: 3
12:08:17.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.533568  3405 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.534849  3405 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.252:36877: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.559471  3477 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } attempt: 1
12:08:17.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.561122  3477 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.252:36877: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.565991  3405 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } attempt: 1
12:08:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.567267  3405 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.252:36877: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.594780  3479 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.594985  3479 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.595027  3479 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.598234  3479 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:08:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.598289  3479 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.598306  3479 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:08:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.598323  3479 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:08:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.598340  3479 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:08:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.598352  3479 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38167/jwks.json
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.602116  3479 master_runner.cc:387] Master server non-default flags:
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.252:36877
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data/info.pb
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.252
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3479
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/logs
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.603102  3479 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.604274  3479 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.610224  3487 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.610306  3490 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.610321  3488 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:17.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.610653  3479 server_base.cc:1061] running on GCE node
12:08:17.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.612386  3479 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.613245  3479 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.614394  3479 hybrid_clock.cc:648] HybridClock initialized: now 1783512497614385 us; error 24 us; skew 500 ppm
12:08:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.616082  3479 webserver.cc:533] Webserver started at http://127.3.63.252:46791/ using document root <none> and password file <none>
12:08:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.616638  3479 fs_manager.cc:362] Metadata directory not provided
12:08:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.616695  3479 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:17.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.616917  3479 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:08:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.618429  3479 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data/instance:
12:08:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f710c09ec775402a90345b9f2cb7e74f"
12:08:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.618914  3479 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal/instance:
12:08:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f710c09ec775402a90345b9f2cb7e74f"
12:08:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.622016  3479 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.005s	sys 0.000s
12:08:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.623890  3496 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.624742  3479 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
12:08:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.624833  3479 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f710c09ec775402a90345b9f2cb7e74f"
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:17 on dist-test-slave-zwgr"
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.624904  3479 fs_report.cc:389] FS layout report
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data/data
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:17.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:17.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:17.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:17.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.629295  3477 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } attempt: 2
12:08:17.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.630733  3477 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.252:36877: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.642416  3405 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } attempt: 2
12:08:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.643795  3405 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.63.252:36877: Network error: Client connection negotiation failed: client connection to 127.3.63.252:36877: connect: Connection refused (error 111)
12:08:17.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.653496  3479 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.654264  3479 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.654445  3479 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:17.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.661367  3479 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.252:36877
12:08:17.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.661396  3548 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.252:36877 every 8 connection(s)
12:08:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.662443  3479 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/data/info.pb
12:08:17.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.665167  3549 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:08:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.667155  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 3479
12:08:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.667268  3327 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/master-2/wal/instance
12:08:17.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.668283  3549 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.677560  3549 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.682713  3549 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } has no permanent_uuid. Determining permanent_uuid...
12:08:17.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.688428  3549 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: Bootstrap starting.
12:08:17.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.690574  3549 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: Neither blocks nor log segments found. Creating new log.
12:08:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.691780  3549 log.cc:826] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: Log is configured to *not* fsync() on all Append() calls
12:08:17.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.693553  3549 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: No bootstrap required, opened a new log
12:08:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.697013  3549 raft_consensus.cc:359] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.697206  3549 raft_consensus.cc:385] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:08:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.697245  3549 raft_consensus.cc:740] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f710c09ec775402a90345b9f2cb7e74f, State: Initialized, Role: FOLLOWER
12:08:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.697636  3549 consensus_queue.cc:260] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.698140  3558 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.698257  3558 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.698951  3549 sys_catalog.cc:565] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: configured and running, proceeding with master startup.
12:08:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.704315  3569 catalog_manager.cc:1594] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:08:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.704411  3569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:08:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.703127  3479 master_runner.cc:187] Error getting master registration for 127.3.63.254:33189: OK, instance_id { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" instance_seqno: 1783512497290472 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:08:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.705127  3479 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:08:17.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.721892  3477 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } attempt: 3
12:08:17.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.728181  3477 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: Bootstrap starting.
12:08:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.730487  3477 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: Neither blocks nor log segments found. Creating new log.
12:08:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.731254  3477 log.cc:826] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: Log is configured to *not* fsync() on all Append() calls
12:08:17.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.733574  3477 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: No bootstrap required, opened a new log
12:08:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.736289  3477 raft_consensus.cc:359] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.736440  3477 raft_consensus.cc:385] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:08:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.736487  3477 raft_consensus.cc:740] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2a52f918a5a4cddb9667c7584fcb515, State: Initialized, Role: FOLLOWER
12:08:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.736953  3477 consensus_queue.cc:260] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.737462  3572 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.737602  3572 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.738142  3477 sys_catalog.cc:565] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: configured and running, proceeding with master startup.
12:08:17.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.742954  3405 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } attempt: 3
12:08:17.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.743050  3583 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:08:17.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.743711  3583 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:08:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.741811  3407 master_runner.cc:187] Error getting master registration for 127.3.63.254:33189: OK, instance_id { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" instance_seqno: 1783512497290472 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:08:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.744097  3407 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:08:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.751797  3405 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: Bootstrap starting.
12:08:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.754523  3405 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: Neither blocks nor log segments found. Creating new log.
12:08:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.756587  3405 log.cc:826] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: Log is configured to *not* fsync() on all Append() calls
12:08:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.759050  3405 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: No bootstrap required, opened a new log
12:08:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.762833  3405 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.763481  3405 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:08:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.763790  3405 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4623f53aa4ed471b852d610ec0c683b4, State: Initialized, Role: FOLLOWER
12:08:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.764536  3405 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.765378  3586 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.765791  3586 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.766170  3405 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: configured and running, proceeding with master startup.
12:08:17.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.771231  3597 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:08:17.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.771586  3597 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:08:17.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.776742  3338 master_runner.cc:264] No leader master found from master 4623f53aa4ed471b852d610ec0c683b4
12:08:17.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.776821  3338 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:08:17.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.786943  3358 tablet.cc:2406] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:08:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.869664  3558 raft_consensus.cc:493] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:08:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.869851  3558 raft_consensus.cc:515] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.870916  3558 leader_election.cc:290] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4623f53aa4ed471b852d610ec0c683b4 (127.3.63.254:33189), a2a52f918a5a4cddb9667c7584fcb515 (127.3.63.253:39047)
12:08:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.872694  3379 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f710c09ec775402a90345b9f2cb7e74f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4623f53aa4ed471b852d610ec0c683b4" is_pre_election: true
12:08:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.872983  3379 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f710c09ec775402a90345b9f2cb7e74f in term 0.
12:08:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.874010  3451 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f710c09ec775402a90345b9f2cb7e74f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2a52f918a5a4cddb9667c7584fcb515" is_pre_election: true
12:08:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.874274  3451 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f710c09ec775402a90345b9f2cb7e74f in term 0.
12:08:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.875105  3498 leader_election.cc:304] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a2a52f918a5a4cddb9667c7584fcb515, f710c09ec775402a90345b9f2cb7e74f; no voters: 
12:08:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.875449  3558 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Leader pre-election won for term 1
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.875522  3558 raft_consensus.cc:493] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.875569  3558 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 0 FOLLOWER]: Advancing to term 1
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.876698  3558 raft_consensus.cc:515] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.877099  3558 leader_election.cc:290] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [CANDIDATE]: Term 1 election: Requested vote from peers 4623f53aa4ed471b852d610ec0c683b4 (127.3.63.254:33189), a2a52f918a5a4cddb9667c7584fcb515 (127.3.63.253:39047)
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.877998  3379 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f710c09ec775402a90345b9f2cb7e74f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4623f53aa4ed471b852d610ec0c683b4"
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.878131  3379 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 0 FOLLOWER]: Advancing to term 1
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.879295  3379 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f710c09ec775402a90345b9f2cb7e74f in term 1.
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.879621  3451 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f710c09ec775402a90345b9f2cb7e74f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2a52f918a5a4cddb9667c7584fcb515"
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.879743  3451 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 0 FOLLOWER]: Advancing to term 1
12:08:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.880949  3451 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f710c09ec775402a90345b9f2cb7e74f in term 1.
12:08:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.881343  3498 leader_election.cc:304] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a2a52f918a5a4cddb9667c7584fcb515, f710c09ec775402a90345b9f2cb7e74f; no voters: 
12:08:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.883663  3558 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 1 FOLLOWER]: Leader election won for term 1
12:08:17.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.883877  3558 raft_consensus.cc:697] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [term 1 LEADER]: Becoming Leader. State: Replica: f710c09ec775402a90345b9f2cb7e74f, State: Running, Role: LEADER
12:08:17.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.884200  3558 consensus_queue.cc:237] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } }
12:08:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.889766  3558 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f710c09ec775402a90345b9f2cb7e74f. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.889885  3558 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: This master's current role is: LEADER
12:08:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.897423  3601 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
12:08:17.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.899802  3601 catalog_manager.cc:1520] Initializing Kudu cluster ID...
12:08:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.905315  3451 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [term 1 FOLLOWER]: Refusing update from remote peer f710c09ec775402a90345b9f2cb7e74f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.905315  3379 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [term 1 FOLLOWER]: Refusing update from remote peer f710c09ec775402a90345b9f2cb7e74f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:17.906412  3502 tablet.cc:2406] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:08:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.906522  3599 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:08:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.907441  3599 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:08:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.910753  3586 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f710c09ec775402a90345b9f2cb7e74f. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.910892  3586 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.910858  3572 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f710c09ec775402a90345b9f2cb7e74f. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.910972  3572 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.917874  3599 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.918009  3599 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: This master's current role is: LEADER
12:08:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.918128  3599 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.918215  3599 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f [sys.catalog]: This master's current role is: LEADER
12:08:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.919245  3572 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.919359  3572 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515 [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.919682  3602 mvcc.cc:204] Tried to move back new op lower bound from 7305267191413219328 to 7305267191348658176. Current Snapshot: MvccSnapshot[applied={T|T < 7305267191413219328}]
12:08:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.919834  3604 mvcc.cc:204] Tried to move back new op lower bound from 7305267191413219328 to 7305267191348658176. Current Snapshot: MvccSnapshot[applied={T|T < 7305267191413219328}]
12:08:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.920635  3586 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f710c09ec775402a90345b9f2cb7e74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4623f53aa4ed471b852d610ec0c683b4" member_type: VOTER last_known_addr { host: "127.3.63.254" port: 33189 } } peers { permanent_uuid: "a2a52f918a5a4cddb9667c7584fcb515" member_type: VOTER last_known_addr { host: "127.3.63.253" port: 39047 } } peers { permanent_uuid: "f710c09ec775402a90345b9f2cb7e74f" member_type: VOTER last_known_addr { host: "127.3.63.252" port: 36877 } } }
12:08:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.920750  3586 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4 [sys.catalog]: This master's current role is: FOLLOWER
12:08:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.921502  3603 mvcc.cc:204] Tried to move back new op lower bound from 7305267191413219328 to 7305267191348658176. Current Snapshot: MvccSnapshot[applied={T|T < 7305267191413219328}]
12:08:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.923292  3601 catalog_manager.cc:1383] Generated new cluster ID: bb0a1fc7dbf44147ba90faf3c5112830
12:08:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.923354  3601 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
12:08:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.943470  3601 catalog_manager.cc:1406] Generated new certificate authority record
12:08:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.944767  3601 catalog_manager.cc:1540] Loading token signing keys...
12:08:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.952842  3601 catalog_manager.cc:6070] T 00000000000000000000000000000000 P f710c09ec775402a90345b9f2cb7e74f: Generated new TSK 0
12:08:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.953941  3601 catalog_manager.cc:1550] Initializing in-progress tserver states...
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:17.959137  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:0
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:18.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.063501  3611 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.063740  3611 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.063828  3611 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:18.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.067430  3611 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:18.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.067575  3611 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.193
12:08:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.071564  3611 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:0
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3611
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.072988  3611 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.074512  3611 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.080840  3616 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.080842  3617 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.080843  3619 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.082056  3611 server_base.cc:1061] running on GCE node
12:08:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.082407  3611 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:18.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.082901  3611 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.084064  3611 hybrid_clock.cc:648] HybridClock initialized: now 1783512498084040 us; error 40 us; skew 500 ppm
12:08:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.085897  3611 webserver.cc:533] Webserver started at http://127.3.63.193:32863/ using document root <none> and password file <none>
12:08:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.086452  3611 fs_manager.cc:362] Metadata directory not provided
12:08:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.086508  3611 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.086722  3611 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:08:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.088292  3611 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/instance:
12:08:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.088800  3611 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal/instance:
12:08:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.091986  3611 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
12:08:18.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.094141  3625 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.095402  3611 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
12:08:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.095502  3611 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.095575  3611 fs_report.cc:389] FS layout report
12:08:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/data
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.117550  3611 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.118189  3611 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.118335  3611 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.118827  3611 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:18.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.119836  3611 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:08:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.119884  3611 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:08:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.119922  3611 ts_tablet_manager.cc:616] Registered 0 tablets
12:08:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.119935  3611 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:08:18.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.128652  3611 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.193:34755
12:08:18.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.128690  3738 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.193:34755 every 8 connection(s)
12:08:18.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.129498  3611 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.135136  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 3611
12:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.135238  3327 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal/instance
12:08:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.138397  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:0
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.138765  3739 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.138809  3740 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.139007  3740 heartbeater.cc:461] Registering TS with master...
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.139523  3740 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.140053  3741 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.140122  3741 heartbeater.cc:461] Registering TS with master...
12:08:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.140285  3741 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.139009  3739 heartbeater.cc:461] Registering TS with master...
12:08:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.140669  3739 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.141129  3512 ts_manager.cc:194] Registered new tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.141863  3369 ts_manager.cc:194] Registered new tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.142635  3512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.193:46691
12:08:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.142832  3437 ts_manager.cc:194] Registered new tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.240070  3746 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.240285  3746 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.240329  3746 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.243752  3746 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.243851  3746 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.194
12:08:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.247637  3746 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:0
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3746
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.248623  3746 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.249737  3746 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.255931  3752 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.256004  3746 server_base.cc:1061] running on GCE node
12:08:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.255931  3754 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.255934  3751 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.256528  3746 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.257095  3746 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.258242  3746 hybrid_clock.cc:648] HybridClock initialized: now 1783512498258218 us; error 35 us; skew 500 ppm
12:08:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.260048  3746 webserver.cc:533] Webserver started at http://127.3.63.194:35517/ using document root <none> and password file <none>
12:08:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.260613  3746 fs_manager.cc:362] Metadata directory not provided
12:08:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.260671  3746 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.260900  3746 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:08:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.262401  3746 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/instance:
12:08:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7df98c593a294597b27ff79721619533"
12:08:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.262892  3746 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal/instance:
12:08:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7df98c593a294597b27ff79721619533"
12:08:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.265959  3746 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.000s	sys 0.005s
12:08:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.268018  3760 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.269011  3746 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.000s	sys 0.002s
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.269109  3746 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7df98c593a294597b27ff79721619533"
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.269199  3746 fs_report.cc:389] FS layout report
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/data
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.287772  3746 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.288560  3746 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.288748  3746 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.289369  3746 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.290405  3746 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:08:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.290455  3746 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:08:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.290508  3746 ts_tablet_manager.cc:616] Registered 0 tablets
12:08:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.290529  3746 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:08:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.299955  3746 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.194:43923
12:08:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.300002  3873 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.194:43923 every 8 connection(s)
12:08:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.301029  3746 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:18.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.305069  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 3746
12:08:18.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.305155  3327 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal/instance
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.308822  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:0
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.310984  3875 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311220  3875 heartbeater.cc:461] Registering TS with master...
12:08:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311410  3876 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311496  3876 heartbeater.cc:461] Registering TS with master...
12:08:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311848  3874 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311836  3876 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311915  3874 heartbeater.cc:461] Registering TS with master...
12:08:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.311836  3875 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.312084  3874 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.312889  3512 ts_manager.cc:194] Registered new tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.312906  3437 ts_manager.cc:194] Registered new tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.313493  3369 ts_manager.cc:194] Registered new tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.313545  3512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.194:49895
12:08:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.416908  3881 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.417243  3881 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.417301  3881 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.421309  3881 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.421458  3881 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.195
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.426201  3881 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:0
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3881
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.427719  3881 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.429083  3881 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.435523  3886 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.435504  3887 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.435603  3881 server_base.cc:1061] running on GCE node
12:08:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:18.435505  3889 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.436347  3881 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:18.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.436879  3881 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:18.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.438048  3881 hybrid_clock.cc:648] HybridClock initialized: now 1783512498438033 us; error 31 us; skew 500 ppm
12:08:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.439874  3881 webserver.cc:533] Webserver started at http://127.3.63.195:45531/ using document root <none> and password file <none>
12:08:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.440460  3881 fs_manager.cc:362] Metadata directory not provided
12:08:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.440521  3881 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.440745  3881 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:08:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.442317  3881 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/instance:
12:08:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.442837  3881 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal/instance:
12:08:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.446473  3881 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.006s	sys 0.000s
12:08:18.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.448865  3895 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.449880  3881 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.449996  3881 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.450098  3881 fs_report.cc:389] FS layout report
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/data
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:18.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.466638  3881 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.467469  3881 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.467667  3881 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:18.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.468324  3881 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.469413  3881 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:08:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.469463  3881 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:08:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.469518  3881 ts_tablet_manager.cc:616] Registered 0 tablets
12:08:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.469539  3881 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:08:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.479452  3881 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.195:46477
12:08:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.479494  4008 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.195:46477 every 8 connection(s)
12:08:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.480654  3881 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.487608  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 3881
12:08:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.487715  3327 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal/instance
12:08:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.491302  4010 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.491561  4010 heartbeater.cc:461] Registering TS with master...
12:08:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.491659  4009 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.491730  4009 heartbeater.cc:461] Registering TS with master...
12:08:18.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.492154  4009 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:18.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.492154  4010 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:18.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.493286  3512 ts_manager.cc:194] Registered new tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:18.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.493300  3369 ts_manager.cc:194] Registered new tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.493974  3512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.195:45839
12:08:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.494345  4011 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.494423  4011 heartbeater.cc:461] Registering TS with master...
12:08:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.494616  4011 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.495328  3437 ts_manager.cc:194] Registered new tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.497068  3327 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
12:08:18.498 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:08:18.499 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

12:08:18.507 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.3.63.254"
      port: 33189
    }
    bound_http_address {
      host: "127.3.63.254"
      port: 36573
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.3.63.253"
      port: 39047
    }
    bound_http_address {
      host: "127.3.63.253"
      port: 35451
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.3.63.252"
      port: 36877
    }
    bound_http_address {
      host: "127.3.63.252"
      port: 46791
    }
  }
}

12:08:18.510 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

12:08:18.512 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.3.63.193"
      port: 34755
    }
    bound_http_address {
      host: "127.3.63.193"
      port: 32863
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.3.63.194"
      port: 43923
    }
    bound_http_address {
      host: "127.3.63.194"
      port: 35517
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.3.63.195"
      port: 46477
    }
    bound_http_address {
      host: "127.3.63.195"
      port: 45531
    }
  }
}

12:08:18.513 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
12:08:18.675 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
12:08:18.740 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
12:08:18.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.745148  3583 catalog_manager.cc:1295] Loaded cluster ID: bb0a1fc7dbf44147ba90faf3c5112830
12:08:18.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.745219  3583 catalog_manager.cc:1588] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: loading cluster ID for follower catalog manager: success
12:08:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.747033  3583 catalog_manager.cc:1610] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: acquiring CA information for follower catalog manager: success
12:08:18.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.748374  3583 catalog_manager.cc:1638] T 00000000000000000000000000000000 P a2a52f918a5a4cddb9667c7584fcb515: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:08:18.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.773228  3597 catalog_manager.cc:1295] Loaded cluster ID: bb0a1fc7dbf44147ba90faf3c5112830
12:08:18.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.773320  3597 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: loading cluster ID for follower catalog manager: success
12:08:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.775792  3597 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: acquiring CA information for follower catalog manager: success
12:08:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:18.777273  3597 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 4623f53aa4ed471b852d610ec0c683b4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:08:18.925 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.63.254:33189(127.3.63.254:33189)] connecting to peer
12:08:18.977 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.63.252:36877(127.3.63.252:36877)] connecting to peer
12:08:18.979 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.63.253:39047(127.3.63.253:39047)] connecting to peer
12:08:18.998 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.63.254:33189(127.3.63.254:33189)] Successfully connected to peer
12:08:18.998 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.3.63.252:36877(127.3.63.252:36877)] Successfully connected to peer
12:08:18.998 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.3.63.253:39047(127.3.63.253:39047)] Successfully connected to peer
12:08:19.093 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.63.253
12:08:19.093 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.63.252
12:08:19.093 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.63.254
12:08:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.145090  3740 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.315285  3875 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:19.326 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xcbf26bf1, L:/127.0.0.1:34762 - R:/127.3.63.253:39047] using SASL/PLAIN
12:08:19.326 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdefa7d07, L:/127.0.0.1:56034 - R:/127.3.63.254:33189] using SASL/PLAIN
12:08:19.328 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xbfb90268, L:/127.0.0.1:54382 - R:/127.3.63.252:36877] using SASL/PLAIN
12:08:19.362 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
12:08:19.363 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=489), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.63.254:33189, [94ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.63.252:36877, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.63.253:39047, [455ms] Sub RPC ConnectToMaster: received response from server master-127.3.63.253:39047: OK, [455ms] Sub RPC ConnectToMaster: received response from server master-127.3.63.254:33189: OK, [456ms] Sub RPC ConnectToMaster: received response from server master-127.3.63.252:36877: OK, deferred=null) after lookup
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.420082  3512 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54382:
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858"
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:08:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
12:08:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:08:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:08:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:19.421603  3512 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
12:08:19.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.464100  3673 tablet_service.cc:1511] Processing CreateTablet for tablet 2e0389dea5a34da1ba9cd502be7a4ee9 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858 [id=ca63ba0571f04016916dac37f726337b]), partition=RANGE (key) PARTITION UNBOUNDED
12:08:19.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.465196  3673 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e0389dea5a34da1ba9cd502be7a4ee9. 1 dirs total, 0 dirs full, 0 dirs failed
12:08:19.479 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table ca63ba0571f04016916dac37f726337b
12:08:19.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.480496  3943 tablet_service.cc:1511] Processing CreateTablet for tablet 2e0389dea5a34da1ba9cd502be7a4ee9 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858 [id=ca63ba0571f04016916dac37f726337b]), partition=RANGE (key) PARTITION UNBOUNDED
12:08:19.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.481449  3943 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e0389dea5a34da1ba9cd502be7a4ee9. 1 dirs total, 0 dirs full, 0 dirs failed
12:08:19.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.485307  4042 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap starting.
12:08:19.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.488016  4042 tablet_bootstrap.cc:654] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Neither blocks nor log segments found. Creating new log.
12:08:19.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.488900  4042 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Log is configured to *not* fsync() on all Append() calls
12:08:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.491304  3808 tablet_service.cc:1511] Processing CreateTablet for tablet 2e0389dea5a34da1ba9cd502be7a4ee9 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858 [id=ca63ba0571f04016916dac37f726337b]), partition=RANGE (key) PARTITION UNBOUNDED
12:08:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.492336  3808 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e0389dea5a34da1ba9cd502be7a4ee9. 1 dirs total, 0 dirs full, 0 dirs failed
12:08:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.493428  4044 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap starting.
12:08:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.495559  4044 tablet_bootstrap.cc:654] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Neither blocks nor log segments found. Creating new log.
12:08:19.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.496454  4010 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.496883  4044 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Log is configured to *not* fsync() on all Append() calls
12:08:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.498742  4044 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: No bootstrap required, opened a new log
12:08:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.498947  4044 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent bootstrapping tablet: real 0.006s	user 0.005s	sys 0.000s
12:08:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.496946  4042 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: No bootstrap required, opened a new log
12:08:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.499982  4042 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent bootstrapping tablet: real 0.015s	user 0.005s	sys 0.000s
12:08:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.503283  4044 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.503785  4044 raft_consensus.cc:385] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:08:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.503677  4042 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.503865  4042 raft_consensus.cc:385] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:08:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.503924  4042 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Initialized, Role: FOLLOWER
12:08:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.504186  4044 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Initialized, Role: FOLLOWER
12:08:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.504421  4042 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.505550  4042 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
12:08:19.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.504757  4044 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.512960  4044 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent starting tablet: real 0.014s	user 0.006s	sys 0.000s
12:08:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.513531  4049 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:08:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.513680  4049 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.514150  4046 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap starting.
12:08:19.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.515451  4049 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.520195  3828 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533" is_pre_election: true
12:08:19.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:19.522276  3896 leader_election.cc:343] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Illegal state: must be running to vote when last-logged opid is not known
12:08:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.523334  3693 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165" is_pre_election: true
12:08:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.523624  3693 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 0.
12:08:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.524008  4046 tablet_bootstrap.cc:654] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Neither blocks nor log segments found. Creating new log.
12:08:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.524214  3899 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.524453  4049 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Leader pre-election won for term 1
12:08:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.524514  4049 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:08:19.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.524550  4049 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 0 FOLLOWER]: Advancing to term 1
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.525642  4049 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.525774  4046 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Log is configured to *not* fsync() on all Append() calls
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.526049  4049 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 1 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.526546  3693 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.526546  3828 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533"
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.526659  3693 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 0 FOLLOWER]: Advancing to term 1
12:08:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:19.526921  3896 leader_election.cc:343] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Illegal state: must be running to vote when last-logged opid is not known
12:08:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.527786  3693 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 1.
12:08:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.528277  3899 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.528492  4049 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 FOLLOWER]: Leader election won for term 1
12:08:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.528570  4049 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 LEADER]: Becoming Leader. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Running, Role: LEADER
12:08:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.528852  4049 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.529088  4046 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: No bootstrap required, opened a new log
12:08:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.529286  4046 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent bootstrapping tablet: real 0.015s	user 0.004s	sys 0.000s
12:08:19.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.531906  4046 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.532298  4046 raft_consensus.cc:385] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:08:19.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.532508  4046 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7df98c593a294597b27ff79721619533, State: Initialized, Role: FOLLOWER
12:08:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.533123  4046 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.534165  4046 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
12:08:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.535300  3512 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195). New cstate: current_term: 1 leader_uuid: "a535e5786752492ba8bd7288d7cc49d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:19.548 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table ca63ba0571f04016916dac37f726337b
12:08:19.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:19.553747  3877 tablet.cc:2406] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:08:19.580 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:19.587 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:19.589 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=null) after lookup
12:08:19.601 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:19.603 [DEBUG - kudu-nio-3] (Connection.java:757) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] Successfully connected to peer
12:08:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:19.632246  3742 tablet.cc:2406] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:08:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.655027  4054 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:08:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.655221  4054 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:19.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.656559  4054 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:19.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.666069  3693 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165" is_pre_election: true
12:08:19.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.666321  3693 raft_consensus.cc:2393] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 7df98c593a294597b27ff79721619533 in current term 1: Already voted for candidate a535e5786752492ba8bd7288d7cc49d9 in this term.
12:08:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.671691  3963 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9" is_pre_election: true
12:08:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.672391  3764 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7df98c593a294597b27ff79721619533; no voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165
12:08:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.672896  4054 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 0 FOLLOWER]: Advancing to term 1
12:08:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.674047  4054 raft_consensus.cc:2749] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
12:08:19.682 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x5abc70b0, L:/127.0.0.1:60372 - R:/127.3.63.195:46477] using TOKEN/null
12:08:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.690506  3693 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Refusing update from remote peer a535e5786752492ba8bd7288d7cc49d9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:08:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.690506  3828 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Refusing update from remote peer a535e5786752492ba8bd7288d7cc49d9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:08:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.691182  4048 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:08:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.691560  4049 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:08:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.701682  4059 mvcc.cc:204] Tried to move back new op lower bound from 7305267198724919296 to 7305267198073217024. Current Snapshot: MvccSnapshot[applied={T|T < 7305267198724919296}]
12:08:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.702356  4060 mvcc.cc:204] Tried to move back new op lower bound from 7305267198724919296 to 7305267198073217024. Current Snapshot: MvccSnapshot[applied={T|T < 7305267198724919296}]
12:08:19.788 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:19.790 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.195:46477
12:08:19.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:19.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:19.792091  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 3881
12:08:19.798 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:19.801 [DEBUG - kudu-nio-3] (Connection.java:239) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] handling channelInactive
12:08:19.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state READY due to: connection closed
12:08:19.802 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: connection closed
12:08:19.822 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:19.822 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:19.823 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test)) after lookup
12:08:19.823 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:19.826 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.826 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.826 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:19.842 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:19.842 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [25ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [28ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [28ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [43ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518))) after lookup
12:08:19.843 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:19.845 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.845 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.846 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.882 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:19.883 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:19.884 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [25ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [28ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [28ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [43ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [48ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [48ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [84ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941))) after lookup
12:08:19.885 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:19.887 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.887 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.887 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.922 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:19.922 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:19.923 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [25ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [28ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [28ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [43ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [48ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [48ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [84ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [86ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [89ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [89ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [124ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862))) after lookup
12:08:19.924 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:19.926 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.926 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:19.926 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:20.081 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:20.081 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:20.082 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=283), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [25ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [28ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [28ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [43ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [48ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [48ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [84ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [86ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [89ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [89ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [124ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [126ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [128ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [128ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [280ms] refreshing cache from master, [280ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [283ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862) -> (continuation of Deferred@2112508076 after retry RPC@1836820088), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862) -> (continuation of Deferred@2112508076 after retry RPC@1836820088))) after lookup
12:08:20.083 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:20.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:20.084 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:20.084 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:20.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:20.841 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:20.842 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1043), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [25ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [28ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [28ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [43ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [48ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [48ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [84ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [86ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [89ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [89ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [124ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [126ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [128ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [128ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [280ms] refreshing cache from master, [280ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [283ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [284ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [286ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [286ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [1040ms] refreshing cache from master, [1040ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1043ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862) -> (continuation of Deferred@2112508076 after retry RPC@1836820088) -> (continuation of Deferred@566158899 after retry RPC@298241050), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862) -> (continuation of Deferred@2112508076 after retry RPC@1836820088) -> (continuation of Deferred@566158899 after retry RPC@298241050))) after lookup
12:08:20.843 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:20.844 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:20.844 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:20.844 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.243901  4073 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.244062  4073 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.244102  4074 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.244230  4074 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.244762  4074 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.245110  4073 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:21.245530  3764 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.245594  3693 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165" is_pre_election: true
12:08:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.245730  3693 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 1.
12:08:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:21.246126  3629 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.246131  3764 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: 
12:08:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.246495  4074 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Leader pre-election won for term 2
12:08:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.246564  4074 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Starting leader election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.246605  4074 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 1 FOLLOWER]: Advancing to term 2
12:08:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:21.246898  3764 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:21.247257  3629 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.247928  4074 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.248278  4074 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 2 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.248775  3693 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.248886  3693 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 1 FOLLOWER]: Advancing to term 2
12:08:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.249274  3828 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533" is_pre_election: true
12:08:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:21.249286  3764 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.249454  3828 raft_consensus.cc:2393] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ce8846e31f454232a6cf4bed41dcb165 in current term 2: Already voted for candidate 7df98c593a294597b27ff79721619533 in this term.
12:08:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.249814  3693 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 2.
12:08:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.249817  3626 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9
12:08:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.250178  4073 raft_consensus.cc:2749] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
12:08:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.250252  3764 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.250413  4074 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Leader election won for term 2
12:08:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.250612  4074 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 LEADER]: Becoming Leader. State: Replica: 7df98c593a294597b27ff79721619533, State: Running, Role: LEADER
12:08:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.250923  4074 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.253494  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 reported cstate change: term changed from 1 to 2, leader changed from a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195) to 7df98c593a294597b27ff79721619533 (127.3.63.194). New cstate: current_term: 2 leader_uuid: "7df98c593a294597b27ff79721619533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: HEALTHY } } }
12:08:21.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:21.813974  3764 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 -> Peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Couldn't send request to peer a535e5786752492ba8bd7288d7cc49d9. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.848220  3693 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Refusing update from remote peer 7df98c593a294597b27ff79721619533: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
12:08:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:21.848827  4095 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
12:08:23.861 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:23.862 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:23.862 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=4063), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [4ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [23ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [25ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [28ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [28ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [43ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [48ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [48ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [84ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [86ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [89ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [89ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [124ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [126ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [128ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [128ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [280ms] refreshing cache from master, [280ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [283ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [284ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [286ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [286ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [1040ms] refreshing cache from master, [1040ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1043ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [1044ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1046ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [1046ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4060ms] refreshing cache from master, [4060ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [4063ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1930840060(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@1e07223e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862) -> (continuation of Deferred@2112508076 after retry RPC@1836820088) -> (continuation of Deferred@566158899 after retry RPC@298241050) -> (continuation of Deferred@899961559 after retry RPC@537602882), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3ef50604 -> wakeup thread Time-limited test -> (continuation of Deferred@14626753 after retry RPC@1546867518) -> (continuation of Deferred@1242133903 after retry RPC@1225659941) -> (continuation of Deferred@1275831960 after retry RPC@594084862) -> (continuation of Deferred@2112508076 after retry RPC@1836820088) -> (continuation of Deferred@566158899 after retry RPC@298241050) -> (continuation of Deferred@899961559 after retry RPC@537602882))) after lookup
12:08:23.863 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:23.864 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] Successfully connected to peer
12:08:23.888 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x97b07a76, L:/127.0.0.1:40874 - R:/127.3.63.194:43923] using TOKEN/null
12:08:23.913 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.195:46477
12:08:23.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:23.915513  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:24.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.018121  4119 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:24.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.018338  4119 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:24.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.018384  4119 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.021780  4119 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.021898  4119 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.195
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.025820  4119 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4119
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:24.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.026881  4119 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:24.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.027962  4119 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.034554  4125 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.034552  4124 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.034646  4119 server_base.cc:1061] running on GCE node
12:08:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:24.034554  4127 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.035156  4119 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.035751  4119 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:24.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.036902  4119 hybrid_clock.cc:648] HybridClock initialized: now 1783512504036878 us; error 38 us; skew 500 ppm
12:08:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.038602  4119 webserver.cc:533] Webserver started at http://127.3.63.195:45531/ using document root <none> and password file <none>
12:08:24.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.039189  4119 fs_manager.cc:362] Metadata directory not provided
12:08:24.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.039494  4119 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.043015  4119 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
12:08:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.044938  4133 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.046222  4119 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
12:08:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.046355  4119 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.046840  4119 fs_report.cc:389] FS layout report
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/data
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.062536  4119 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.063482  4119 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.063670  4119 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.064318  4119 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.065722  4142 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.069240  4119 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.069319  4119 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
12:08:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.069391  4119 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.071097  4119 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.071161  4119 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
12:08:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.071521  4142 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap starting.
12:08:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.082036  4119 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.195:46477
12:08:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.082072  4249 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.195:46477 every 8 connection(s)
12:08:24.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.083106  4119 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.086522  4142 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Log is configured to *not* fsync() on all Append() calls
12:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.091272  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 4119
12:08:24.091 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.097119  4142 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.098133  4142 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap complete.
12:08:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.098639  4252 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.098850  4252 heartbeater.cc:461] Registering TS with master...
12:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.099174  4251 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.099256  4251 heartbeater.cc:461] Registering TS with master...
12:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.098577  4250 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.099485  4251 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.099545  4250 heartbeater.cc:461] Registering TS with master...
12:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.099709  4250 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:24.101 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:24.102 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.194:43923
12:08:24.103 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:08:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.101584  3369 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.101888  3513 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.099483  4252 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.103185  3513 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.195:44041
12:08:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.103340  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 3746
12:08:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.105587  3441 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.105888  4142 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent bootstrapping tablet: real 0.035s	user 0.016s	sys 0.008s
12:08:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.107987  4142 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.108311  4142 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Initialized, Role: FOLLOWER
12:08:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.108768  4142 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.109474  4251 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:24.109596  4142 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
12:08:24.112 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] handling channelInactive
12:08:24.112 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state READY due to: connection closed
12:08:24.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:24.114 [DEBUG - Time-limited test] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.115 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.115 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.115 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.121 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:24.122 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:24.122 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test)) after lookup
12:08:24.123 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.124 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.124 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.124 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.141 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:24.141 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:24.142 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571))) after lookup
12:08:24.143 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.144 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.144 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.144 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.181 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:24.181 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:24.182 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [31ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [31ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565))) after lookup
12:08:24.183 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.184 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.184 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.185 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.222 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:24.222 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:24.223 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [31ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [31ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [72ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [105ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198))) after lookup
12:08:24.223 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.225 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.225 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.225 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.701 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:24.702 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:24.702 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=588), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [31ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [31ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [72ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [105ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [110ms] sending RPC to server 7df98c593a294597b27ff79721619533, [112ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [112ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [585ms] refreshing cache from master, [585ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [588ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198) -> (continuation of Deferred@1283198602 after retry RPC@169275692), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198) -> (continuation of Deferred@1283198602 after retry RPC@169275692))) after lookup
12:08:24.703 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.704 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.704 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.704 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:24.841 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:24.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=728), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [31ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [31ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [72ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [105ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [110ms] sending RPC to server 7df98c593a294597b27ff79721619533, [112ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [112ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [585ms] refreshing cache from master, [585ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [588ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [590ms] sending RPC to server 7df98c593a294597b27ff79721619533, [591ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [591ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [725ms] refreshing cache from master, [725ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [727ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198) -> (continuation of Deferred@1283198602 after retry RPC@169275692) -> (continuation of Deferred@1964217304 after retry RPC@1211208647), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198) -> (continuation of Deferred@1283198602 after retry RPC@169275692) -> (continuation of Deferred@1964217304 after retry RPC@1211208647))) after lookup
12:08:24.842 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:24.843 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:24.844 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:25.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.411682  4263 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 7df98c593a294597b27ff79721619533)
12:08:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.411855  4263 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.412261  4263 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:25.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.413573  3626 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.417085  4204 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9" is_pre_election: true
12:08:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.417325  4204 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ce8846e31f454232a6cf4bed41dcb165 in term 1.
12:08:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.417929  3629 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.418133  4263 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Leader pre-election won for term 3
12:08:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.418198  4263 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 7df98c593a294597b27ff79721619533)
12:08:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.418243  4263 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 2 FOLLOWER]: Advancing to term 3
12:08:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.419111  4263 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.419514  4263 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 3 election: Requested vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.419850  4204 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.419975  4204 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 1 FOLLOWER]: Advancing to term 3
12:08:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.420583  3626 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:25.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.421666  4204 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce8846e31f454232a6cf4bed41dcb165 in term 3.
12:08:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.421967  3629 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.422129  4263 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 FOLLOWER]: Leader election won for term 3
12:08:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.422333  4263 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 LEADER]: Becoming Leader. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Running, Role: LEADER
12:08:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.422643  4263 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.424958  3513 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 reported cstate change: term changed from 2 to 3, leader changed from 7df98c593a294597b27ff79721619533 (127.3.63.194) to ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193). New cstate: current_term: 3 leader_uuid: "ce8846e31f454232a6cf4bed41dcb165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:25.441 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:25.441 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:25.442 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1328), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [31ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [31ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [72ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [105ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [110ms] sending RPC to server 7df98c593a294597b27ff79721619533, [112ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [112ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [585ms] refreshing cache from master, [585ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [588ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [590ms] sending RPC to server 7df98c593a294597b27ff79721619533, [591ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [591ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [725ms] refreshing cache from master, [725ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [727ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [729ms] sending RPC to server 7df98c593a294597b27ff79721619533, [730ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [731ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [1325ms] refreshing cache from master, [1325ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1328ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@496873843(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6435c8b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198) -> (continuation of Deferred@1283198602 after retry RPC@169275692) -> (continuation of Deferred@1964217304 after retry RPC@1211208647) -> (continuation of Deferred@758256094 after retry RPC@1062667351), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@48d2b067 -> wakeup thread Time-limited test -> (continuation of Deferred@993992763 after retry RPC@1714683571) -> (continuation of Deferred@1704144298 after retry RPC@481071565) -> (continuation of Deferred@1644736371 after retry RPC@441608198) -> (continuation of Deferred@1283198602 after retry RPC@169275692) -> (continuation of Deferred@1964217304 after retry RPC@1211208647) -> (continuation of Deferred@758256094 after retry RPC@1062667351))) after lookup
12:08:25.442 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.443 [DEBUG - kudu-nio-2] (Connection.java:757) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] Successfully connected to peer
12:08:25.465 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x9c9d5f43, L:/127.0.0.1:46742 - R:/127.3.63.193:34755] using TOKEN/null
12:08:25.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.470170  3626 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 -> Peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Couldn't send request to peer 7df98c593a294597b27ff79721619533. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:25.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.470217  4204 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Refusing update from remote peer ce8846e31f454232a6cf4bed41dcb165: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
12:08:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.470863  4263 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
12:08:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.474663  4284 mvcc.cc:204] Tried to move back new op lower bound from 7305267215979941888 to 7305267205126610944. Current Snapshot: MvccSnapshot[applied={T|T < 7305267215947321344}]
12:08:25.494 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.194:43923
12:08:25.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:08:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.496584  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:43923
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35517
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.595345  4291 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.595582  4291 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.595626  4291 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.598807  4291 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.598904  4291 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.194
12:08:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.602653  4291 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:43923
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35517
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4291
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.603792  4291 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.605091  4291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.611389  4296 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.611408  4299 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:25.611441  4297 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.611660  4291 server_base.cc:1061] running on GCE node
12:08:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.611970  4291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.612452  4291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.613596  4291 hybrid_clock.cc:648] HybridClock initialized: now 1783512505613582 us; error 32 us; skew 500 ppm
12:08:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.615123  4291 webserver.cc:533] Webserver started at http://127.3.63.194:35517/ using document root <none> and password file <none>
12:08:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.615629  4291 fs_manager.cc:362] Metadata directory not provided
12:08:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.615692  4291 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.618613  4291 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
12:08:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.620337  4305 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.621194  4291 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.621289  4291 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7df98c593a294597b27ff79721619533"
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.621662  4291 fs_report.cc:389] FS layout report
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/data
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.631131  4291 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.631693  4291 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.631861  4291 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.632467  4291 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.633632  4312 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.636770  4291 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.636826  4291 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
12:08:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.636893  4291 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.638281  4291 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.638322  4291 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:08:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.638610  4312 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap starting.
12:08:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.648146  4291 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.194:43923
12:08:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.648183  4419 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.194:43923 every 8 connection(s)
12:08:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.649072  4291 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.652122  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 4291
12:08:25.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.653117  4312 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Log is configured to *not* fsync() on all Append() calls
12:08:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.658891  4422 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.659133  4422 heartbeater.cc:461] Registering TS with master...
12:08:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.659729  4422 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:25.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.661803  3441 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.663257  4420 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.663341  4420 heartbeater.cc:461] Registering TS with master...
12:08:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.663573  4420 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:25.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.664469  3368 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:25.668 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:25.668 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.193:34755
12:08:25.669 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.667484  4421 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.667682  4421 heartbeater.cc:461] Registering TS with master...
12:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.667955  4421 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.668955  3511 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.669481  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 3611
12:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.669795  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.194:56213
12:08:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.674757  4312 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.675146  4312 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap complete.
12:08:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.676426  4312 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent bootstrapping tablet: real 0.038s	user 0.018s	sys 0.008s
12:08:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.678824  4312 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.679232  4312 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7df98c593a294597b27ff79721619533, State: Initialized, Role: FOLLOWER
12:08:25.679 [DEBUG - kudu-nio-2] (Connection.java:239) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] handling channelInactive
12:08:25.680 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.679870  4312 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:25.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state READY due to: connection closed
12:08:25.680 [DEBUG - Time-limited test] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.682 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.682 [DEBUG - kudu-nio-3] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.682 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.681981  4421 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:25.682286  4312 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent starting tablet: real 0.006s	user 0.002s	sys 0.003s
12:08:25.701 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:25.702 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:25.702 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [2ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [2ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@513070789(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@19e60cfc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6eee55d4 -> wakeup thread Time-limited test)) after lookup
12:08:25.703 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.704 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.704 [DEBUG - kudu-nio-4] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.704 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.721 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:25.721 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:25.722 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [2ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [2ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [22ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [24ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [24ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [41ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@513070789(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@19e60cfc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6eee55d4 -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400))) after lookup
12:08:25.722 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.724 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.724 [DEBUG - kudu-nio-5] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.724 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.741 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:25.742 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:25.742 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [2ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [2ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [22ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [24ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [24ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [41ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [42ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [44ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [44ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [58ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [61ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@513070789(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@19e60cfc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6eee55d4 -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615))) after lookup
12:08:25.743 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.744 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.744 [DEBUG - kudu-nio-6] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.744 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.781 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:25.781 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:25.782 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=101), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [2ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [2ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [22ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [24ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [24ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [41ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [42ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [44ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [44ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [58ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [61ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [62ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [64ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [64ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [101ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@513070789(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@19e60cfc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615) -> (continuation of Deferred@523624060 after retry RPC@334021410), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6eee55d4 -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615) -> (continuation of Deferred@523624060 after retry RPC@334021410))) after lookup
12:08:25.782 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.783 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.784 [DEBUG - kudu-nio-7] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.784 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:25.841 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:25.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=161), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [2ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [2ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [22ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [24ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [24ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [41ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [42ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [44ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [44ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [58ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [61ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [62ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [64ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [64ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [101ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [102ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [104ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [104ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [160ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@513070789(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@19e60cfc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615) -> (continuation of Deferred@523624060 after retry RPC@334021410) -> (continuation of Deferred@1871141055 after retry RPC@983787454), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6eee55d4 -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615) -> (continuation of Deferred@523624060 after retry RPC@334021410) -> (continuation of Deferred@1871141055 after retry RPC@983787454))) after lookup
12:08:25.842 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:25.843 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.843 [DEBUG - kudu-nio-0] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:25.843 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:26.994680  4434 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader ce8846e31f454232a6cf4bed41dcb165)
12:08:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:26.994850  4434 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:26.996017  4434 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:26.996788  4139 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:26.997624  4139 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.001096  4374 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533" is_pre_election: true
12:08:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.001379  4374 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 2.
12:08:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.002045  4136 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9; no voters: ce8846e31f454232a6cf4bed41dcb165
12:08:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.002270  4434 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Leader pre-election won for term 4
12:08:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.002341  4434 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Starting leader election (detected failure of leader ce8846e31f454232a6cf4bed41dcb165)
12:08:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.002383  4434 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 3 FOLLOWER]: Advancing to term 4
12:08:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.003165  4434 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.003571  4434 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 4 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.003957  4374 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533"
12:08:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.004098  4374 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 2 FOLLOWER]: Advancing to term 4
12:08:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.004567  4139 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:27.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.005893  4374 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 4.
12:08:27.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.006215  4136 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9; no voters: ce8846e31f454232a6cf4bed41dcb165
12:08:27.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.006422  4434 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 FOLLOWER]: Leader election won for term 4
12:08:27.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.006608  4434 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 LEADER]: Becoming Leader. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Running, Role: LEADER
12:08:27.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.006937  4434 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.009205  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 reported cstate change: term changed from 3 to 4, leader changed from ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193) to a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195). New cstate: current_term: 4 leader_uuid: "a535e5786752492ba8bd7288d7cc49d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:27.141 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:27.141 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:27.142 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1461), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [2ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [2ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [21ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [22ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [24ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [24ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [41ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [42ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [44ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [44ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [58ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [61ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [62ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [64ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [64ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [101ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [102ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [104ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [104ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [160ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [162ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [163ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [163ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [1458ms] refreshing cache from master, [1458ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1461ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@513070789(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@19e60cfc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615) -> (continuation of Deferred@523624060 after retry RPC@334021410) -> (continuation of Deferred@1871141055 after retry RPC@983787454) -> (continuation of Deferred@552950876 after retry RPC@21643627), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6eee55d4 -> wakeup thread Time-limited test -> (continuation of Deferred@1882872038 after retry RPC@1818686400) -> (continuation of Deferred@550369501 after retry RPC@2079258615) -> (continuation of Deferred@523624060 after retry RPC@334021410) -> (continuation of Deferred@1871141055 after retry RPC@983787454) -> (continuation of Deferred@552950876 after retry RPC@21643627))) after lookup
12:08:27.142 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.143 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] Successfully connected to peer
12:08:27.161 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x23e2b01f, L:/127.0.0.1:36302 - R:/127.3.63.195:46477] using TOKEN/null
12:08:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.166066  4374 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Refusing update from remote peer a535e5786752492ba8bd7288d7cc49d9: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
12:08:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.166162  4139 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 -> Peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Couldn't send request to peer ce8846e31f454232a6cf4bed41dcb165. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.166769  4434 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
12:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.175439  4456 mvcc.cc:204] Tried to move back new op lower bound from 7305267229345320960 to 7305267228702855168. Current Snapshot: MvccSnapshot[applied={T|T < 7305267229345320960}]
12:08:27.189 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.193:34755
12:08:27.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.190140  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:34755
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32863
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.286726  4463 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.286947  4463 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.287014  4463 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.290421  4463 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.290539  4463 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.193
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.294893  4463 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:34755
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32863
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4463
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.296161  4463 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:27.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.297412  4463 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.304427  4469 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.304435  4468 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:27.304582  4471 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.305020  4463 server_base.cc:1061] running on GCE node
12:08:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.305359  4463 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.305864  4463 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.307029  4463 hybrid_clock.cc:648] HybridClock initialized: now 1783512507307016 us; error 29 us; skew 500 ppm
12:08:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.308874  4463 webserver.cc:533] Webserver started at http://127.3.63.193:32863/ using document root <none> and password file <none>
12:08:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.309381  4463 fs_manager.cc:362] Metadata directory not provided
12:08:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.309444  4463 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.312826  4463 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
12:08:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.314671  4477 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:27.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.315606  4463 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.315697  4463 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.316059  4463 fs_report.cc:389] FS layout report
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/data
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.335263  4463 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.335906  4463 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.336056  4463 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.336548  4463 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.337827  4484 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.341226  4463 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.341287  4463 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
12:08:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.341337  4463 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.342754  4463 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.342808  4463 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:08:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.343125  4484 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap starting.
12:08:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.353895  4463 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.193:34755
12:08:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.353912  4591 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.193:34755 every 8 connection(s)
12:08:27.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.355225  4463 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.359875  4484 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Log is configured to *not* fsync() on all Append() calls
12:08:27.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.364527  4594 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:27.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.364753  4594 heartbeater.cc:461] Registering TS with master...
12:08:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.365099  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 4463
12:08:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.365250  4594 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:27.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:27.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.367166  4593 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:27.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.367261  4593 heartbeater.cc:461] Registering TS with master...
12:08:27.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.367483  4593 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.368353  3513 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.368582  4592 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.368666  4592 heartbeater.cc:461] Registering TS with master...
12:08:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.368824  4592 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.369251  3513 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.193:40139
12:08:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.369644  3369 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.369644  3441 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:27.375 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.376048  4484 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.376471  4484 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap complete.
12:08:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.377454  4484 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent bootstrapping tablet: real 0.035s	user 0.017s	sys 0.012s
12:08:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.379860  4484 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.380250  4484 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Initialized, Role: FOLLOWER
12:08:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.380741  4484 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.381618  4484 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
12:08:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.382578  4593 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:27.383 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.195:46477
12:08:27.383 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:27.383665  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 4119
12:08:27.393 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] handling channelInactive
12:08:27.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:27.393 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state READY due to: connection closed
12:08:27.394 [DEBUG - Time-limited test] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.395 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.395 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.396 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.400 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:27.401 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:27.401 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test)) after lookup
12:08:27.401 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.402 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.403 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.403 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.421 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:27.422 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:27.422 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541))) after lookup
12:08:27.422 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.423 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.424 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.424 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.461 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:27.462 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:27.462 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [28ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653))) after lookup
12:08:27.463 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.464 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.464 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.464 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.501 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:27.501 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:27.502 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=107), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [28ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [70ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [70ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [107ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524))) after lookup
12:08:27.502 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.504 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.504 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.504 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.581 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:27.581 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:27.581 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=187), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [28ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [70ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [70ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [107ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [108ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [186ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731))) after lookup
12:08:27.582 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:27.583 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.583 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:27.583 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.341 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:28.342 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:28.342 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=948), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [28ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [70ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [70ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [107ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [108ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [186ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [188ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [189ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [189ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [944ms] refreshing cache from master, [945ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [947ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731) -> (continuation of Deferred@135210099 after retry RPC@1082554621), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731) -> (continuation of Deferred@135210099 after retry RPC@1082554621))) after lookup
12:08:28.343 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:28.344 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.345 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.381 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:28.381 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:28.382 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=987), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [28ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [70ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [70ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [107ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [108ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [186ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [188ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [189ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [189ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [944ms] refreshing cache from master, [945ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [947ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [948ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [950ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [951ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [984ms] refreshing cache from master, [984ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [987ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731) -> (continuation of Deferred@135210099 after retry RPC@1082554621) -> (continuation of Deferred@377085450 after retry RPC@1699417512), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731) -> (continuation of Deferred@135210099 after retry RPC@1082554621) -> (continuation of Deferred@377085450 after retry RPC@1699417512))) after lookup
12:08:28.382 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:28.383 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.383 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.384 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.689013  4605 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.689177  4605 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.690179  4605 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:28.691238  4309 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:28.692200  4309 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.695343  4546 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165" is_pre_election: true
12:08:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.695655  4546 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 3.
12:08:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.696301  4309 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.696556  4605 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Leader pre-election won for term 5
12:08:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.696621  4605 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Starting leader election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.696662  4605 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 4 FOLLOWER]: Advancing to term 5
12:08:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.697508  4605 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.697894  4605 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 5 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.698292  4546 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.698429  4546 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 3 FOLLOWER]: Advancing to term 5
12:08:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:28.699004  4309 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.700343  4546 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 5.
12:08:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.700650  4309 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.700821  4605 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 FOLLOWER]: Leader election won for term 5
12:08:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.701045  4605 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 LEADER]: Becoming Leader. State: Replica: 7df98c593a294597b27ff79721619533, State: Running, Role: LEADER
12:08:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.701366  4605 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:28.703769  3513 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 reported cstate change: term changed from 4 to 5, leader changed from a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195) to 7df98c593a294597b27ff79721619533 (127.3.63.194). New cstate: current_term: 5 leader_uuid: "7df98c593a294597b27ff79721619533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: HEALTHY } } }
12:08:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:29.233484  4546 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Refusing update from remote peer 7df98c593a294597b27ff79721619533: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
12:08:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:29.234287  4622 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
12:08:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:29.243695  4309 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 -> Peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Couldn't send request to peer a535e5786752492ba8bd7288d7cc49d9. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:30.362 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:30.362 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:30.363 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=16, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2968), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [2ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [7ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [9ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [9ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [28ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [70ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [70ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [107ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [108ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [186ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [188ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [189ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [189ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [944ms] refreshing cache from master, [945ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [947ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [948ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [950ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [951ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [984ms] refreshing cache from master, [984ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [987ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [988ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [990ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [990ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2964ms] refreshing cache from master, [2965ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [2967ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1999206277(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@79b29aea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731) -> (continuation of Deferred@135210099 after retry RPC@1082554621) -> (continuation of Deferred@377085450 after retry RPC@1699417512) -> (continuation of Deferred@1243293091 after retry RPC@225833742), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7fe75e -> wakeup thread Time-limited test -> (continuation of Deferred@1842386078 after retry RPC@150002541) -> (continuation of Deferred@333216904 after retry RPC@502916653) -> (continuation of Deferred@211685620 after retry RPC@1585731524) -> (continuation of Deferred@472079028 after retry RPC@1625625731) -> (continuation of Deferred@135210099 after retry RPC@1082554621) -> (continuation of Deferred@377085450 after retry RPC@1699417512) -> (continuation of Deferred@1243293091 after retry RPC@225833742))) after lookup
12:08:30.363 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.364 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] Successfully connected to peer
12:08:30.383 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xc8925310, L:/127.0.0.1:49040 - R:/127.3.63.194:43923] using TOKEN/null
12:08:30.402 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.195:46477
12:08:30.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.403172  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.505191  4647 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.505421  4647 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.505481  4647 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.508903  4647 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.509029  4647 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.195
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.512969  4647 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4647
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.514379  4647 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.515506  4647 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:30.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.521584  4652 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.521648  4653 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.521824  4647 server_base.cc:1061] running on GCE node
12:08:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:30.521586  4655 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.522502  4647 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.523057  4647 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.524224  4647 hybrid_clock.cc:648] HybridClock initialized: now 1783512510524205 us; error 29 us; skew 500 ppm
12:08:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.526017  4647 webserver.cc:533] Webserver started at http://127.3.63.195:45531/ using document root <none> and password file <none>
12:08:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.526558  4647 fs_manager.cc:362] Metadata directory not provided
12:08:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.526618  4647 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:30.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.529901  4647 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
12:08:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.531946  4661 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.532953  4647 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.533061  4647 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.533488  4647 fs_report.cc:389] FS layout report
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/data
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:30.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.547724  4647 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.548418  4647 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.548596  4647 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.549170  4647 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.550462  4668 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:30.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.553555  4647 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:30.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.553611  4647 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.001s	sys 0.000s
12:08:30.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.553669  4647 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.555084  4647 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.555128  4647 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:08:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.555480  4668 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap starting.
12:08:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.564492  4647 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.195:46477
12:08:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.564548  4775 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.195:46477 every 8 connection(s)
12:08:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.565518  4647 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.570623  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 4647
12:08:30.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.574050  4668 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Log is configured to *not* fsync() on all Append() calls
12:08:30.581 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:30.582 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.194:43923
12:08:30.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:08:30.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.582804  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 4291
12:08:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.583258  4776 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.583564  4776 heartbeater.cc:461] Registering TS with master...
12:08:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.583734  4777 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.583859  4777 heartbeater.cc:461] Registering TS with master...
12:08:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.584144  4776 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.584170  4778 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.584820  4778 heartbeater.cc:461] Registering TS with master...
12:08:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.584992  4778 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.586010  3441 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.584144  4777 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:30.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.587622  3368 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.588032  3511 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.588930  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.195:49633
12:08:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.590718  4668 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.591091  4668 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap complete.
12:08:30.592 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] handling channelInactive
12:08:30.592 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state READY due to: connection closed
12:08:30.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.592221  4668 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent bootstrapping tablet: real 0.037s	user 0.022s	sys 0.010s
12:08:30.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:30.593 [DEBUG - Time-limited test] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.594 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.595 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.594740  4668 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:30.595 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.595077  4668 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Initialized, Role: FOLLOWER
12:08:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.595553  4668 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.596246  4777 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:30.596401  4668 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
12:08:30.601 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:30.601 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:30.602 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2021672098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@5a4f353f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@223eff1d -> wakeup thread Time-limited test)) after lookup
12:08:30.602 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.603 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.604 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.604 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.621 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:30.621 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:30.621 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2021672098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@5a4f353f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@223eff1d -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375))) after lookup
12:08:30.622 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.623 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.623 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.623 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.661 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:30.662 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:30.662 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [30ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [30ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2021672098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@5a4f353f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@223eff1d -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724))) after lookup
12:08:30.663 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.664 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.664 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.664 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:30.761 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:30.761 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=168), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [30ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [30ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [71ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [167ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2021672098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@5a4f353f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724) -> (continuation of Deferred@938734511 after retry RPC@2127347336), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@223eff1d -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724) -> (continuation of Deferred@938734511 after retry RPC@2127347336))) after lookup
12:08:30.762 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.763 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.763 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.763 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.941 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:30.941 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:30.942 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=348), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [30ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [30ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [71ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [167ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [168ms] sending RPC to server 7df98c593a294597b27ff79721619533, [170ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [170ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [345ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [348ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2021672098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@5a4f353f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724) -> (continuation of Deferred@938734511 after retry RPC@2127347336) -> (continuation of Deferred@2046517558 after retry RPC@1658370175), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@223eff1d -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724) -> (continuation of Deferred@938734511 after retry RPC@2127347336) -> (continuation of Deferred@2046517558 after retry RPC@1658370175))) after lookup
12:08:30.942 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:30.943 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.943 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:30.943 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.902000  4789 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 7df98c593a294597b27ff79721619533)
12:08:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.902137  4789 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.903131  4789 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:31.904065  4478 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:31.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:31.904983  4478 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.907725  4730 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9" is_pre_election: true
12:08:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.907972  4730 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ce8846e31f454232a6cf4bed41dcb165 in term 4.
12:08:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.908607  4481 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.908841  4789 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Leader pre-election won for term 6
12:08:31.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.908905  4789 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 7df98c593a294597b27ff79721619533)
12:08:31.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.908931  4789 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 5 FOLLOWER]: Advancing to term 6
12:08:31.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.909651  4789 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:31.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.910018  4789 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 6 election: Requested vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:31.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.910348  4730 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:31.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.910465  4730 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 4 FOLLOWER]: Advancing to term 6
12:08:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:31.910982  4478 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.912134  4730 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce8846e31f454232a6cf4bed41dcb165 in term 6.
12:08:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.912446  4481 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.912611  4789 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 FOLLOWER]: Leader election won for term 6
12:08:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.912848  4789 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 LEADER]: Becoming Leader. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Running, Role: LEADER
12:08:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.913137  4789 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:31.915480  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 reported cstate change: term changed from 5 to 6, leader changed from 7df98c593a294597b27ff79721619533 (127.3.63.194) to ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193). New cstate: current_term: 6 leader_uuid: "ce8846e31f454232a6cf4bed41dcb165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:32.021 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:32.021 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:32.023 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1429), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [2ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [9ms] sending RPC to server 7df98c593a294597b27ff79721619533, [11ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [11ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server 7df98c593a294597b27ff79721619533, [30ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [30ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [68ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [69ms] sending RPC to server 7df98c593a294597b27ff79721619533, [71ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [71ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [167ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [168ms] sending RPC to server 7df98c593a294597b27ff79721619533, [170ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [170ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [345ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [348ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [349ms] sending RPC to server 7df98c593a294597b27ff79721619533, [350ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [350ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [1425ms] refreshing cache from master, [1425ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1427ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2021672098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@5a4f353f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724) -> (continuation of Deferred@938734511 after retry RPC@2127347336) -> (continuation of Deferred@2046517558 after retry RPC@1658370175) -> (continuation of Deferred@1883645005 after retry RPC@1467642234), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@223eff1d -> wakeup thread Time-limited test -> (continuation of Deferred@978492367 after retry RPC@1874943375) -> (continuation of Deferred@1146428756 after retry RPC@563940724) -> (continuation of Deferred@938734511 after retry RPC@2127347336) -> (continuation of Deferred@2046517558 after retry RPC@1658370175) -> (continuation of Deferred@1883645005 after retry RPC@1467642234))) after lookup
12:08:32.023 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:32.024 [DEBUG - kudu-nio-1] (Connection.java:757) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] Successfully connected to peer
12:08:32.041 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x911495cb, L:/127.0.0.1:46826 - R:/127.3.63.193:34755] using TOKEN/null
12:08:32.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.044929  4730 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Refusing update from remote peer ce8846e31f454232a6cf4bed41dcb165: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 22. (index mismatch)
12:08:32.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.045167  4478 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 -> Peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Couldn't send request to peer 7df98c593a294597b27ff79721619533. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:32.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.045594  4794 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
12:08:32.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.052708  4807 mvcc.cc:204] Tried to move back new op lower bound from 7305267249329426432 to 7305267248798560256. Current Snapshot: MvccSnapshot[applied={T|T < 7305267249329426432}]
12:08:32.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.053990  4809 mvcc.cc:204] Tried to move back new op lower bound from 7305267249329426432 to 7305267248798560256. Current Snapshot: MvccSnapshot[applied={T|T < 7305267249329426432}]
12:08:32.070 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.194:43923
12:08:32.071 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.072211  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:43923
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35517
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.177448  4817 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.177650  4817 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.177695  4817 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.181020  4817 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.181125  4817 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.194
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.184929  4817 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:43923
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35517
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4817
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.185962  4817 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:32.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.186960  4817 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.193141  4825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.193145  4822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:32.193146  4823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.193346  4817 server_base.cc:1061] running on GCE node
12:08:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.193717  4817 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.194202  4817 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.195348  4817 hybrid_clock.cc:648] HybridClock initialized: now 1783512512195329 us; error 29 us; skew 500 ppm
12:08:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.197058  4817 webserver.cc:533] Webserver started at http://127.3.63.194:35517/ using document root <none> and password file <none>
12:08:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.197580  4817 fs_manager.cc:362] Metadata directory not provided
12:08:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.197638  4817 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.200804  4817 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.000s	sys 0.003s
12:08:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.202672  4831 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.203639  4817 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.002s
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.203742  4817 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7df98c593a294597b27ff79721619533"
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.204165  4817 fs_report.cc:389] FS layout report
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/data
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:32.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.218482  4817 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.219162  4817 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.219337  4817 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.219942  4817 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.221230  4838 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.224411  4817 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.224464  4817 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
12:08:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.224522  4817 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.225977  4817 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.226030  4817 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
12:08:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.226356  4838 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap starting.
12:08:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.237442  4817 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.194:43923
12:08:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.237450  4945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.194:43923 every 8 connection(s)
12:08:32.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.238837  4817 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.247345  4838 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Log is configured to *not* fsync() on all Append() calls
12:08:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.248173  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 4817
12:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.250262  4947 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.250412  4946 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.250500  4946 heartbeater.cc:461] Registering TS with master...
12:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.250500  4947 heartbeater.cc:461] Registering TS with master...
12:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.251075  4946 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.251075  4947 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:32.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.253067  3368 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:32.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.253068  3511 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:32.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.253968  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.194:34523
12:08:32.262 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.262024  4948 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.262145  4948 heartbeater.cc:461] Registering TS with master...
12:08:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.262339  4948 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:32.263 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.193:34755
12:08:32.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:08:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.263772  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 4463
12:08:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.264106  3441 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.268103  4838 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.268538  4838 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap complete.
12:08:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.269495  4838 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent bootstrapping tablet: real 0.043s	user 0.028s	sys 0.007s
12:08:32.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:32.270 [DEBUG - kudu-nio-1] (Connection.java:239) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] handling channelInactive
12:08:32.270 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state READY due to: connection closed
12:08:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.272238  4838 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.272581  4838 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7df98c593a294597b27ff79721619533, State: Initialized, Role: FOLLOWER
12:08:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.273067  4838 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:32.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.273923  4838 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
12:08:32.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:32.274063  4947 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:32.278 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:32.280 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.280 [DEBUG - kudu-nio-2] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.280 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.301 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:32.301 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:32.302 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test)) after lookup
12:08:32.302 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:32.303 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.303 [DEBUG - kudu-nio-3] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.303 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.321 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:32.321 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:32.321 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064))) after lookup
12:08:32.322 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:32.323 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.323 [DEBUG - kudu-nio-4] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.323 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.366 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:32.367 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:32.367 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=96), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [51ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [53ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [53ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [96ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131))) after lookup
12:08:32.368 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:32.369 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.369 [DEBUG - kudu-nio-5] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.369 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.542 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:32.543 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:32.543 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=272), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [51ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [53ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [53ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [96ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [98ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [99ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [99ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [272ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165))) after lookup
12:08:32.544 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:32.545 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.545 [DEBUG - kudu-nio-6] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:32.545 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.182 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:33.182 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:33.182 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=911), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [51ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [53ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [53ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [96ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [98ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [99ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [99ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [272ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [274ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [275ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [275ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [908ms] refreshing cache from master, [909ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [911ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340))) after lookup
12:08:33.183 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:33.184 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.184 [DEBUG - kudu-nio-7] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.184 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.221 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:33.221 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:33.222 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=951), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [51ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [53ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [53ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [96ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [98ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [99ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [99ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [272ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [274ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [275ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [275ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [908ms] refreshing cache from master, [909ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [911ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [913ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [914ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [914ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [948ms] refreshing cache from master, [948ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [951ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340) -> (continuation of Deferred@496463224 after retry RPC@167431106), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340) -> (continuation of Deferred@496463224 after retry RPC@167431106))) after lookup
12:08:33.222 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:33.223 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.223 [DEBUG - kudu-nio-0] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.223 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.521 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:33.522 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:33.522 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=15, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1251), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [51ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [53ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [53ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [96ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [98ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [99ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [99ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [272ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [274ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [275ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [275ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [908ms] refreshing cache from master, [909ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [911ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [913ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [914ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [914ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [948ms] refreshing cache from master, [948ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [951ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [952ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [953ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [953ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [1249ms] refreshing cache from master, [1249ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1251ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340) -> (continuation of Deferred@496463224 after retry RPC@167431106) -> (continuation of Deferred@855818089 after retry RPC@1828312592), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340) -> (continuation of Deferred@496463224 after retry RPC@167431106) -> (continuation of Deferred@855818089 after retry RPC@1828312592))) after lookup
12:08:33.523 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:33.523 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.524 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.524 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.569418  4960 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader ce8846e31f454232a6cf4bed41dcb165)
12:08:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.569536  4960 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.570565  4960 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:33.571141  4665 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:33.572098  4665 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.576218  4900 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533" is_pre_election: true
12:08:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.576511  4900 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 5.
12:08:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.577313  4662 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9; no voters: ce8846e31f454232a6cf4bed41dcb165
12:08:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.577521  4960 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Leader pre-election won for term 7
12:08:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.577581  4960 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Starting leader election (detected failure of leader ce8846e31f454232a6cf4bed41dcb165)
12:08:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.577618  4960 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 6 FOLLOWER]: Advancing to term 7
12:08:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.578349  4960 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.578680  4960 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 7 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.579023  4900 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533"
12:08:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.579141  4900 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 5 FOLLOWER]: Advancing to term 7
12:08:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:33.579762  4665 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.580788  4900 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 7.
12:08:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.581096  4662 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9; no voters: ce8846e31f454232a6cf4bed41dcb165
12:08:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.581239  4960 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 FOLLOWER]: Leader election won for term 7
12:08:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.581427  4960 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 LEADER]: Becoming Leader. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Running, Role: LEADER
12:08:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.581700  4960 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:33.584120  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 reported cstate change: term changed from 6 to 7, leader changed from ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193) to a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195). New cstate: current_term: 7 leader_uuid: "a535e5786752492ba8bd7288d7cc49d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:34.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:34.029661  4665 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 -> Peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Couldn't send request to peer ce8846e31f454232a6cf4bed41dcb165. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:34.072141  4900 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Refusing update from remote peer a535e5786752492ba8bd7288d7cc49d9: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
12:08:34.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:34.072844  4960 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
12:08:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.422097  4665 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 -> Peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Couldn't send request to peer ce8846e31f454232a6cf4bed41dcb165. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:08:36.721 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:36.721 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:36.722 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=17, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=4451), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [10ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [10ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [32ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [33ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [33ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [50ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [51ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [53ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [53ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [96ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [98ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [99ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [99ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [269ms] refreshing cache from master, [269ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [272ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [274ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [275ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [275ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [908ms] refreshing cache from master, [909ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [911ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [913ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [914ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [914ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [948ms] refreshing cache from master, [948ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [951ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [952ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [953ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [953ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [1249ms] refreshing cache from master, [1249ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1251ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [1252ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1254ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [1254ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [4448ms] refreshing cache from master, [4448ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [4451ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1831507537(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@7cd5f774 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340) -> (continuation of Deferred@496463224 after retry RPC@167431106) -> (continuation of Deferred@855818089 after retry RPC@1828312592) -> (continuation of Deferred@897572088 after retry RPC@1243743284), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2198593f -> wakeup thread Time-limited test -> (continuation of Deferred@1378557619 after retry RPC@832169064) -> (continuation of Deferred@248761981 after retry RPC@1880609131) -> (continuation of Deferred@142880543 after retry RPC@1188445165) -> (continuation of Deferred@476161526 after retry RPC@1797254340) -> (continuation of Deferred@496463224 after retry RPC@167431106) -> (continuation of Deferred@855818089 after retry RPC@1828312592) -> (continuation of Deferred@897572088 after retry RPC@1243743284))) after lookup
12:08:36.722 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:36.723 [DEBUG - kudu-nio-2] (Connection.java:757) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] Successfully connected to peer
12:08:36.739 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xa8120fa4, L:/127.0.0.1:40160 - R:/127.3.63.195:46477] using TOKEN/null
12:08:36.756 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.193:34755
12:08:36.756 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.757030  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:34755
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32863
12:08:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.855531  5001 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.855733  5001 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.855779  5001 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.859154  5001 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.859266  5001 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.193
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.863049  5001 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:34755
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32863
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5001
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.864105  5001 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.865152  5001 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.871584  5006 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.871589  5009 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:36.871622  5007 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.871709  5001 server_base.cc:1061] running on GCE node
12:08:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.872138  5001 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.872689  5001 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.873826  5001 hybrid_clock.cc:648] HybridClock initialized: now 1783512516873808 us; error 36 us; skew 500 ppm
12:08:36.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.875416  5001 webserver.cc:533] Webserver started at http://127.3.63.193:32863/ using document root <none> and password file <none>
12:08:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.875923  5001 fs_manager.cc:362] Metadata directory not provided
12:08:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.875974  5001 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:36.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.879052  5001 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
12:08:36.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.880873  5015 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:36.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.881848  5001 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.881945  5001 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.882309  5001 fs_report.cc:389] FS layout report
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/data
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.900660  5001 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.901314  5001 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.901463  5001 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.902161  5001 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:36.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.903437  5022 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.906653  5001 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.906733  5001 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
12:08:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.906833  5001 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.908434  5001 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.908494  5001 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
12:08:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.908849  5022 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap starting.
12:08:36.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.918674  5001 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.193:34755
12:08:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.918708  5129 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.193:34755 every 8 connection(s)
12:08:36.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.919867  5001 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:36.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.923177  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 5001
12:08:36.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:36.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.924297  5022 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Log is configured to *not* fsync() on all Append() calls
12:08:36.934 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:36.934 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.195:46477
12:08:36.935 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:36.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.935281  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 4647
12:08:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.937180  5130 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.937450  5130 heartbeater.cc:461] Registering TS with master...
12:08:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.938099  5130 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.938556  5131 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.938655  5131 heartbeater.cc:461] Registering TS with master...
12:08:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.938862  5131 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:36.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.940001  3511 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.940202  3369 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.940927  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.193:40879
12:08:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.941084  5132 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.941149  5132 heartbeater.cc:461] Registering TS with master...
12:08:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.941314  5132 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.942147  3437 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:36.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:36.944 [DEBUG - kudu-nio-2] (Connection.java:239) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] handling channelInactive
12:08:36.945 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state READY due to: connection closed
12:08:36.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.946671  5022 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:36.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.947026  5022 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap complete.
12:08:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.947913  5022 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent bootstrapping tablet: real 0.039s	user 0.016s	sys 0.020s
12:08:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.950165  5022 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.950484  5022 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Initialized, Role: FOLLOWER
12:08:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.950918  5022 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.951624  5131 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:36.951802  5022 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent starting tablet: real 0.004s	user 0.002s	sys 0.002s
12:08:36.959 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:36.959 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:36.960 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:36.960 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:36.981 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:36.981 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:36.982 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=3, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test)) after lookup
12:08:36.982 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:36.983 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:36.983 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:36.984 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.000 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:37.001 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:37.001 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=5, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [38ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [40ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [56ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877))) after lookup
12:08:37.002 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:37.002 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.003 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.003 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.061 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:37.062 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:37.062 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=7, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=117), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [38ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [40ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [56ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [57ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [59ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [59ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [117ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432))) after lookup
12:08:37.063 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:37.064 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.064 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.064 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.201 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:37.201 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:37.202 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=9, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=257), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [38ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [40ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [56ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [57ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [59ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [59ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [117ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [118ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [120ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [256ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114))) after lookup
12:08:37.202 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:37.203 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.204 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.204 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.421 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:37.421 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:37.422 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=11, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=477), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [38ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [40ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [56ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [57ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [59ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [59ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [117ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [118ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [120ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [256ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [258ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [260ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [260ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [474ms] refreshing cache from master, [474ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [477ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114) -> (continuation of Deferred@984890806 after retry RPC@2050041363), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114) -> (continuation of Deferred@984890806 after retry RPC@2050041363))) after lookup
12:08:37.422 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:37.423 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.423 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.423 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.722 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:37.722 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:37.722 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=13, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=777), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [38ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [40ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [56ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [57ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [59ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [59ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [117ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [118ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [120ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [256ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [258ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [260ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [260ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [474ms] refreshing cache from master, [474ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [477ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [478ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [479ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [479ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [775ms] refreshing cache from master, [775ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [777ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114) -> (continuation of Deferred@984890806 after retry RPC@2050041363) -> (continuation of Deferred@1430434745 after retry RPC@649070268), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114) -> (continuation of Deferred@984890806 after retry RPC@2050041363) -> (continuation of Deferred@1430434745 after retry RPC@649070268))) after lookup
12:08:37.723 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:37.724 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.724 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:37.724 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.256199  5143 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.256335  5143 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:38.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.257313  5143 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:38.258363  4835 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:38.259140  4835 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.262070  5084 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165" is_pre_election: true
12:08:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.262305  5084 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 6.
12:08:38.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.262909  4835 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:38.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.263115  5143 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Leader pre-election won for term 8
12:08:38.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.263173  5143 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Starting leader election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:38.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.263211  5143 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 7 FOLLOWER]: Advancing to term 8
12:08:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.263989  5143 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.264314  5143 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 8 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.264655  5084 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.264787  5084 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 6 FOLLOWER]: Advancing to term 8
12:08:38.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:38.265341  4835 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.266439  5084 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 8.
12:08:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.266727  4835 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.266904  5143 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 FOLLOWER]: Leader election won for term 8
12:08:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.267081  5143 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 LEADER]: Becoming Leader. State: Replica: 7df98c593a294597b27ff79721619533, State: Running, Role: LEADER
12:08:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.267347  5143 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.269573  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 reported cstate change: term changed from 7 to 8, leader changed from a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195) to 7df98c593a294597b27ff79721619533 (127.3.63.194). New cstate: current_term: 8 leader_uuid: "7df98c593a294597b27ff79721619533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: HEALTHY } } }
12:08:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:38.703663  4835 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 -> Peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Couldn't send request to peer a535e5786752492ba8bd7288d7cc49d9. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:38.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.778258  5084 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Refusing update from remote peer 7df98c593a294597b27ff79721619533: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
12:08:38.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:38.778959  5143 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
12:08:40.941 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:40.941 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:40.941 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=15, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3996), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [16ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [16ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [37ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [38ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [40ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [40ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [56ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [57ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [59ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [59ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [117ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [118ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [120ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [120ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [256ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [258ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [260ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [260ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [474ms] refreshing cache from master, [474ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [477ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [478ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [479ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [479ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [775ms] refreshing cache from master, [775ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [777ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [779ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [780ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [780ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [3994ms] refreshing cache from master, [3994ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [3996ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@2079786223(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@212ebc01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114) -> (continuation of Deferred@984890806 after retry RPC@2050041363) -> (continuation of Deferred@1430434745 after retry RPC@649070268) -> (continuation of Deferred@671183083 after retry RPC@2031089329), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44ae493a -> wakeup thread Time-limited test -> (continuation of Deferred@1202189594 after retry RPC@1911980877) -> (continuation of Deferred@1475910057 after retry RPC@307336432) -> (continuation of Deferred@450918402 after retry RPC@76061114) -> (continuation of Deferred@984890806 after retry RPC@2050041363) -> (continuation of Deferred@1430434745 after retry RPC@649070268) -> (continuation of Deferred@671183083 after retry RPC@2031089329))) after lookup
12:08:40.942 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:40.943 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] Successfully connected to peer
12:08:40.957 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x03df4f2b, L:/127.0.0.1:55938 - R:/127.3.63.194:43923] using TOKEN/null
12:08:40.973 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.195:46477
12:08:40.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:40.974116  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:41.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.075955  5182 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:41.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.076159  5182 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:41.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.076216  5182 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.079638  5182 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.079759  5182 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.195
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.083544  5182 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5182
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.084919  5182 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:41.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.086018  5182 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.092201  5188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.092209  5190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.092414  5182 server_base.cc:1061] running on GCE node
12:08:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:41.092221  5187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.092849  5182 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:41.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.093402  5182 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:41.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.094553  5182 hybrid_clock.cc:648] HybridClock initialized: now 1783512521094528 us; error 36 us; skew 500 ppm
12:08:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.096330  5182 webserver.cc:533] Webserver started at http://127.3.63.195:45531/ using document root <none> and password file <none>
12:08:41.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.096882  5182 fs_manager.cc:362] Metadata directory not provided
12:08:41.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.096941  5182 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.100713  5182 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
12:08:41.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.102674  5196 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.103698  5182 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.103806  5182 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.104244  5182 fs_report.cc:389] FS layout report
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/data
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.114605  5182 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.115271  5182 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.115475  5182 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.116041  5182 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.117226  5203 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.120474  5182 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.120548  5182 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
12:08:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.120612  5182 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:41.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.121978  5182 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:41.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.122020  5182 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:08:41.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.122303  5203 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap starting.
12:08:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.131845  5182 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.195:46477
12:08:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.131863  5310 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.195:46477 every 8 connection(s)
12:08:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.132759  5182 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.138875  5203 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Log is configured to *not* fsync() on all Append() calls
12:08:41.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.139837  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 5182
12:08:41.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:41.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.143447  5312 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:41.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.143697  5312 heartbeater.cc:461] Registering TS with master...
12:08:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.144292  5312 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.145952  5313 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.146046  5313 heartbeater.cc:461] Registering TS with master...
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.146214  5313 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.146247  5311 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.146333  5311 heartbeater.cc:461] Registering TS with master...
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.146368  3511 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.146510  5311 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:41.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.147254  3441 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:41.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.147255  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.195:46909
12:08:41.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.147385  3368 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:41.150 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:41.150 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.194:43923
12:08:41.151 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:08:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.151538  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 4817
12:08:41.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:41.157 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] handling channelInactive
12:08:41.157 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state READY due to: connection closed
12:08:41.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.161458  5203 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:41.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.161831  5203 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap complete.
12:08:41.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.162662  5203 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent bootstrapping tablet: real 0.041s	user 0.018s	sys 0.015s
12:08:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.164793  5203 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.165129  5203 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Initialized, Role: FOLLOWER
12:08:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.165598  5203 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:41.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.166333  5312 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:41.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:41.166435  5203 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent starting tablet: real 0.004s	user 0.006s	sys 0.000s
12:08:41.178 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:41.180 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.180 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.180 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.201 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:41.201 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:41.202 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=3, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 7df98c593a294597b27ff79721619533, [23ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [23ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [44ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1732509930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6178d8cd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3f39f4 -> wakeup thread Time-limited test)) after lookup
12:08:41.202 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:41.203 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.203 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.203 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.220 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:41.220 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:41.221 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=5, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=63), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 7df98c593a294597b27ff79721619533, [23ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [23ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [44ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server 7df98c593a294597b27ff79721619533, [46ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [46ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [63ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1732509930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6178d8cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3f39f4 -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181))) after lookup
12:08:41.221 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:41.222 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.222 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.223 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.300 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:41.301 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:41.301 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=7, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=143), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 7df98c593a294597b27ff79721619533, [23ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [23ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [44ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server 7df98c593a294597b27ff79721619533, [46ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [46ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [63ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [64ms] sending RPC to server 7df98c593a294597b27ff79721619533, [65ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [66ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [143ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1732509930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6178d8cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3f39f4 -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024))) after lookup
12:08:41.302 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:41.303 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.303 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.303 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.421 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:41.422 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:41.422 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=9, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=264), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 7df98c593a294597b27ff79721619533, [23ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [23ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [44ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server 7df98c593a294597b27ff79721619533, [46ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [46ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [63ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [64ms] sending RPC to server 7df98c593a294597b27ff79721619533, [65ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [66ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [143ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [144ms] sending RPC to server 7df98c593a294597b27ff79721619533, [146ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [146ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [261ms] refreshing cache from master, [261ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [264ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1732509930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6178d8cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024) -> (continuation of Deferred@102781985 after retry RPC@265244456), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3f39f4 -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024) -> (continuation of Deferred@102781985 after retry RPC@265244456))) after lookup
12:08:41.423 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:41.424 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.424 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.424 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.961 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:41.961 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:41.961 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=11, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=803), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 7df98c593a294597b27ff79721619533, [23ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [23ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [44ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server 7df98c593a294597b27ff79721619533, [46ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [46ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [63ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [64ms] sending RPC to server 7df98c593a294597b27ff79721619533, [65ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [66ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [143ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [144ms] sending RPC to server 7df98c593a294597b27ff79721619533, [146ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [146ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [261ms] refreshing cache from master, [261ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [264ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [265ms] sending RPC to server 7df98c593a294597b27ff79721619533, [267ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [267ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [801ms] refreshing cache from master, [801ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [803ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1732509930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6178d8cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024) -> (continuation of Deferred@102781985 after retry RPC@265244456) -> (continuation of Deferred@1871750282 after retry RPC@450035807), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3f39f4 -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024) -> (continuation of Deferred@102781985 after retry RPC@265244456) -> (continuation of Deferred@1871750282 after retry RPC@450035807))) after lookup
12:08:41.962 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:41.963 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.963 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:41.963 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7df98c593a294597b27ff79721619533(127.3.63.194:43923) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923
12:08:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.472872  5326 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 7df98c593a294597b27ff79721619533)
12:08:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.473003  5326 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.474089  5326 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:42.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:42.474869  5016 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:42.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:42.475677  5016 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.478778  5265 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9" is_pre_election: true
12:08:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.479065  5265 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ce8846e31f454232a6cf4bed41dcb165 in term 7.
12:08:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.479712  5019 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.479933  5326 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Leader pre-election won for term 9
12:08:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.479995  5326 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 7df98c593a294597b27ff79721619533)
12:08:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.480032  5326 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 8 FOLLOWER]: Advancing to term 9
12:08:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.494307  5326 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.494817  5326 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 9 election: Requested vote from peers a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.495332  5265 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "ce8846e31f454232a6cf4bed41dcb165" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.495523  5265 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 7 FOLLOWER]: Advancing to term 9
12:08:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:42.496004  5016 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111)
12:08:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.502022  5265 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce8846e31f454232a6cf4bed41dcb165 in term 9.
12:08:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.502470  5019 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a535e5786752492ba8bd7288d7cc49d9, ce8846e31f454232a6cf4bed41dcb165; no voters: 7df98c593a294597b27ff79721619533
12:08:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.502658  5326 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 FOLLOWER]: Leader election won for term 9
12:08:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.502866  5326 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 LEADER]: Becoming Leader. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Running, Role: LEADER
12:08:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.503160  5326 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.505843  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 reported cstate change: term changed from 8 to 9, leader changed from 7df98c593a294597b27ff79721619533 (127.3.63.194) to ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193). New cstate: current_term: 9 leader_uuid: "ce8846e31f454232a6cf4bed41dcb165" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:42.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.888108  5265 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Refusing update from remote peer ce8846e31f454232a6cf4bed41dcb165: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
12:08:42.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:42.888867  5326 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
12:08:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:42.897111  5016 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 -> Peer 7df98c593a294597b27ff79721619533 (127.3.63.194:43923): Couldn't send request to peer 7df98c593a294597b27ff79721619533. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.194:43923: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:44.581 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:44.581 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:44.581 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=13, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=3424), Traces: [0ms] sending RPC to server 7df98c593a294597b27ff79721619533, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 7df98c593a294597b27ff79721619533, [23ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [23ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [44ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [45ms] sending RPC to server 7df98c593a294597b27ff79721619533, [46ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [46ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [63ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [64ms] sending RPC to server 7df98c593a294597b27ff79721619533, [65ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [66ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [143ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [144ms] sending RPC to server 7df98c593a294597b27ff79721619533, [146ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [146ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [261ms] refreshing cache from master, [261ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [264ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [265ms] sending RPC to server 7df98c593a294597b27ff79721619533, [267ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [267ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [801ms] refreshing cache from master, [801ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [803ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [805ms] sending RPC to server 7df98c593a294597b27ff79721619533, [806ms] received response from server 7df98c593a294597b27ff79721619533: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [806ms] delaying RPC due to: Network error: Failed to connect to peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923): Connection refused: /127.3.63.194:43923, [3421ms] refreshing cache from master, [3421ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [3424ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@1732509930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@6178d8cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024) -> (continuation of Deferred@102781985 after retry RPC@265244456) -> (continuation of Deferred@1871750282 after retry RPC@450035807) -> (continuation of Deferred@678703319 after retry RPC@707202174), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3f39f4 -> wakeup thread Time-limited test -> (continuation of Deferred@574580495 after retry RPC@1354566181) -> (continuation of Deferred@1860124268 after retry RPC@1452015024) -> (continuation of Deferred@102781985 after retry RPC@265244456) -> (continuation of Deferred@1871750282 after retry RPC@450035807) -> (continuation of Deferred@678703319 after retry RPC@707202174))) after lookup
12:08:44.582 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:44.583 [DEBUG - kudu-nio-1] (Connection.java:757) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] Successfully connected to peer
12:08:44.598 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x84e0700f, L:/127.0.0.1:44024 - R:/127.3.63.193:34755] using TOKEN/null
12:08:44.614 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.194:43923
12:08:44.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:08:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.615549  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:43923
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35517
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.717944  5364 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.718168  5364 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.718231  5364 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.721807  5364 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.721941  5364 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.194
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.725981  5364 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.194:43923
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.194
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35517
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5364
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.194
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/logs
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.727459  5364 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.728586  5364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:44.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.736366  5372 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:44.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.736466  5370 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:44.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:44.736393  5369 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.737075  5364 server_base.cc:1061] running on GCE node
12:08:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.737497  5364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:44.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.738133  5364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:44.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.739303  5364 hybrid_clock.cc:648] HybridClock initialized: now 1783512524739279 us; error 31 us; skew 500 ppm
12:08:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.741127  5364 webserver.cc:533] Webserver started at http://127.3.63.194:35517/ using document root <none> and password file <none>
12:08:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.741648  5364 fs_manager.cc:362] Metadata directory not provided
12:08:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.741699  5364 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.745168  5364 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
12:08:44.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.747081  5378 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.748082  5364 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.748175  5364 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7df98c593a294597b27ff79721619533"
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.748553  5364 fs_report.cc:389] FS layout report
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/wal
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/data
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:44.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:44.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:44.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.763734  5364 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.764333  5364 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.764474  5364 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:44.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.764961  5364 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.766135  5385 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.769454  5364 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.769519  5364 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
12:08:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.769572  5364 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:44.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.771054  5364 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:44.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.771098  5364 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
12:08:44.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.771519  5385 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap starting.
12:08:44.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.781692  5364 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.194:43923
12:08:44.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.781718  5492 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.194:43923 every 8 connection(s)
12:08:44.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.782680  5364 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-1/data/info.pb
12:08:44.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.787793  5385 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Log is configured to *not* fsync() on all Append() calls
12:08:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.791345  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 5364
12:08:44.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.797169  5494 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.797416  5494 heartbeater.cc:461] Registering TS with master...
12:08:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.798004  5494 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:44.799 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:44.800 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.193:34755
12:08:44.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:08:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.799818  3511 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.800773  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.194:32955
12:08:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.801023  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 5001
12:08:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.802069  5495 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.802166  5495 heartbeater.cc:461] Registering TS with master...
12:08:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.802608  5495 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.803551  3441 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:44.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.805688  5493 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:44.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.805774  5493 heartbeater.cc:461] Registering TS with master...
12:08:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.805951  5493 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.806744  3368 ts_manager.cc:194] Re-registered known tserver with Master: 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:44.808 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:44.808 [DEBUG - kudu-nio-1] (Connection.java:239) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] handling channelInactive
12:08:44.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state READY due to: connection closed
12:08:44.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.811662  5385 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:44.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.811987  5385 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Bootstrap complete.
12:08:44.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.812793  5385 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent bootstrapping tablet: real 0.042s	user 0.034s	sys 0.004s
12:08:44.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.814736  5385 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:44.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.815049  5385 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7df98c593a294597b27ff79721619533, State: Initialized, Role: FOLLOWER
12:08:44.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.815448  5385 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:44.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.816031  5494 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:44.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:44.816234  5385 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533: Time spent starting tablet: real 0.003s	user 0.004s	sys 0.000s
12:08:44.818 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:44.819 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.820 [DEBUG - kudu-nio-2] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.820 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:44.841 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:44.841 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=3, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [12ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [12ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [32ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@202195722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@34925813 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2bd497a7 -> wakeup thread Time-limited test)) after lookup
12:08:44.842 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:44.843 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.843 [DEBUG - kudu-nio-3] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.843 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.860 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:44.861 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:44.861 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=5, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [12ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [12ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [32ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [33ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [35ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [35ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [52ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@202195722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@34925813 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2bd497a7 -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454))) after lookup
12:08:44.861 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:44.862 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.862 [DEBUG - kudu-nio-4] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.863 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.941 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:44.941 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:44.942 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=7, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=133), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [12ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [12ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [32ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [33ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [35ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [35ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [52ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [53ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [54ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [55ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [133ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@202195722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@34925813 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454) -> (continuation of Deferred@1960973470 after retry RPC@571768320), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2bd497a7 -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454) -> (continuation of Deferred@1960973470 after retry RPC@571768320))) after lookup
12:08:44.942 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:44.943 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.944 [DEBUG - kudu-nio-5] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:44.944 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:45.180 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:45.181 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)[L]]}]
12:08:45.181 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=9, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=372), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [12ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [12ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [32ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [33ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [35ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [35ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [52ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [53ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [54ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [55ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [133ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [134ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [136ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [136ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [372ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@202195722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@34925813 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454) -> (continuation of Deferred@1960973470 after retry RPC@571768320) -> (continuation of Deferred@1400479978 after retry RPC@1385967867), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2bd497a7 -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454) -> (continuation of Deferred@1960973470 after retry RPC@571768320) -> (continuation of Deferred@1400479978 after retry RPC@1385967867))) after lookup
12:08:45.181 [DEBUG - kudu-nio-1] (Connection.java:748) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] connecting to peer
12:08:45.183 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:45.183 [DEBUG - kudu-nio-6] (Connection.java:709) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] cleaning up while in state CONNECTING due to: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:45.183 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755
12:08:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.114176  5506 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader ce8846e31f454232a6cf4bed41dcb165)
12:08:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.114315  5506 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.115324  5506 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:46.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.115967  5200 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:46.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.117069  5200 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:46.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.120748  5447 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533" is_pre_election: true
12:08:46.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.121022  5447 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 8.
12:08:46.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.121678  5197 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9; no voters: ce8846e31f454232a6cf4bed41dcb165
12:08:46.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.121905  5506 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Leader pre-election won for term 10
12:08:46.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.121978  5506 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Starting leader election (detected failure of leader ce8846e31f454232a6cf4bed41dcb165)
12:08:46.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.122017  5506 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 9 FOLLOWER]: Advancing to term 10
12:08:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.123011  5506 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.123432  5506 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 10 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), 7df98c593a294597b27ff79721619533 (127.3.63.194:43923)
12:08:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.123754  5447 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "a535e5786752492ba8bd7288d7cc49d9" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "7df98c593a294597b27ff79721619533"
12:08:46.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.123870  5447 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 8 FOLLOWER]: Advancing to term 10
12:08:46.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.124411  5200 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111)
12:08:46.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.125515  5447 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a535e5786752492ba8bd7288d7cc49d9 in term 10.
12:08:46.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.125793  5197 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, a535e5786752492ba8bd7288d7cc49d9; no voters: ce8846e31f454232a6cf4bed41dcb165
12:08:46.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.125942  5506 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 10 FOLLOWER]: Leader election won for term 10
12:08:46.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.126116  5506 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 10 LEADER]: Becoming Leader. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Running, Role: LEADER
12:08:46.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.126389  5506 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:46.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.128628  3511 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 reported cstate change: term changed from 9 to 10, leader changed from ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193) to a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195). New cstate: current_term: 10 leader_uuid: "a535e5786752492ba8bd7288d7cc49d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: UNKNOWN } } }
12:08:46.181 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:46.181 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:46.181 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=11, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1372), Traces: [0ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [1ms] delaying RPC due to: Illegal state: connection is terminated, [10ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [12ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [12ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [32ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [33ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [35ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [35ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [52ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [53ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [54ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [55ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [133ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [134ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [136ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [136ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [372ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [373ms] sending RPC to server ce8846e31f454232a6cf4bed41dcb165, [375ms] received response from server ce8846e31f454232a6cf4bed41dcb165: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [375ms] delaying RPC due to: Network error: Failed to connect to peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755): Connection refused: /127.3.63.193:34755, [1370ms] refreshing cache from master, [1370ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1372ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@202195722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@34925813 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454) -> (continuation of Deferred@1960973470 after retry RPC@571768320) -> (continuation of Deferred@1400479978 after retry RPC@1385967867) -> (continuation of Deferred@1983499673 after retry RPC@1167142989), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2bd497a7 -> wakeup thread Time-limited test -> (continuation of Deferred@1784725394 after retry RPC@1703306454) -> (continuation of Deferred@1960973470 after retry RPC@571768320) -> (continuation of Deferred@1400479978 after retry RPC@1385967867) -> (continuation of Deferred@1983499673 after retry RPC@1167142989))) after lookup
12:08:46.182 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:46.183 [DEBUG - kudu-nio-7] (Connection.java:757) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] Successfully connected to peer
12:08:46.204 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x2a30756a, L:/127.0.0.1:58850 - R:/127.3.63.195:46477] using TOKEN/null
12:08:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.207553  5447 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Refusing update from remote peer a535e5786752492ba8bd7288d7cc49d9: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
12:08:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.207739  5200 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 -> Peer ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755): Couldn't send request to peer ce8846e31f454232a6cf4bed41dcb165. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.193:34755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.208153  5511 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
12:08:46.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.213886  5524 mvcc.cc:204] Tried to move back new op lower bound from 7305267307340435456 to 7305267307015921664. Current Snapshot: MvccSnapshot[applied={T|T < 7305267307340435456}]
12:08:46.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.214180  5526 mvcc.cc:204] Tried to move back new op lower bound from 7305267307340435456 to 7305267307015921664. Current Snapshot: MvccSnapshot[applied={T|T < 7305267307340435456}]
12:08:46.226 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.193:34755
12:08:46.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.227427  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:34755
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32863
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.324218  5532 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.324440  5532 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.324491  5532 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:46.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.327781  5532 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.328012  5532 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.193
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.332013  5532 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.193:34755
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.193
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32863
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5532
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.193
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/logs
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.333060  5532 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.334071  5532 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.340035  5538 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.340063  5540 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:46.340060  5537 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.340454  5532 server_base.cc:1061] running on GCE node
12:08:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.340796  5532 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:46.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.341288  5532 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.342453  5532 hybrid_clock.cc:648] HybridClock initialized: now 1783512526342440 us; error 22 us; skew 500 ppm
12:08:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.344101  5532 webserver.cc:533] Webserver started at http://127.3.63.193:32863/ using document root <none> and password file <none>
12:08:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.344606  5532 fs_manager.cc:362] Metadata directory not provided
12:08:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.344663  5532 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.347977  5532 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
12:08:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.349787  5546 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.350693  5532 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.350792  5532 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.351179  5532 fs_report.cc:389] FS layout report
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/wal
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/data
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.366640  5532 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:46.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.367306  5532 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:46.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.367506  5532 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:46.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.368068  5532 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:46.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.369223  5553 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.372385  5532 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.372450  5532 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
12:08:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.372515  5532 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.373978  5532 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.374024  5532 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
12:08:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.374307  5553 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap starting.
12:08:46.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.384075  5532 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.193:34755
12:08:46.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.384100  5663 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.193:34755 every 8 connection(s)
12:08:46.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.385063  5532 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-0/data/info.pb
12:08:46.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.390093  5553 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Log is configured to *not* fsync() on all Append() calls
12:08:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.392565  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 5532
12:08:46.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.395413  5666 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.395587  5666 heartbeater.cc:461] Registering TS with master...
12:08:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.396129  5666 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.396391  5665 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.396474  5665 heartbeater.cc:461] Registering TS with master...
12:08:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.396658  5665 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.396140  5664 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.397372  5664 heartbeater.cc:461] Registering TS with master...
12:08:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.397528  5664 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.398396  3368 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.398478  3513 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.399338  3513 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.193:40811
12:08:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.400388  3437 ts_manager.cc:194] Re-registered known tserver with Master: ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755)
12:08:46.402 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:46.403 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.63.195:46477
12:08:46.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:46.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.403863  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 5182
12:08:46.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:46.412 [DEBUG - kudu-nio-7] (Connection.java:239) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] handling channelInactive
12:08:46.412 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state READY due to: connection closed
12:08:46.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.418226  5553 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:46.418 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:46.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.418565  5553 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Bootstrap complete.
12:08:46.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.419 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.419716  5553 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent bootstrapping tablet: real 0.046s	user 0.030s	sys 0.011s
12:08:46.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.421836  5553 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:46.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.422148  5553 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce8846e31f454232a6cf4bed41dcb165, State: Initialized, Role: FOLLOWER
12:08:46.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.422520  5553 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:46.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.423146  5665 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:46.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:46.423303  5553 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165: Time spent starting tablet: real 0.003s	user 0.003s	sys 0.001s
12:08:46.440 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:46.440 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:46.441 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=3, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [7ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [7ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@731617576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@67c4c74e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1db0b3a3 -> wakeup thread Time-limited test)) after lookup
12:08:46.441 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:46.442 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.442 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.442 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.461 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:46.461 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:46.461 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=5, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [7ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [7ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [48ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@731617576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@67c4c74e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1db0b3a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952))) after lookup
12:08:46.462 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:46.462 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.463 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.463 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.520 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:46.521 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:46.521 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=7, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [7ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [7ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [48ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [49ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [51ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [51ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@731617576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@67c4c74e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1db0b3a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802))) after lookup
12:08:46.521 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:46.522 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.522 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.522 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.781 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:46.781 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:46.782 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=9, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=369), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [7ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [7ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [48ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [49ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [51ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [51ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [109ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [366ms] refreshing cache from master, [366ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [368ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@731617576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@67c4c74e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802) -> (continuation of Deferred@1612615514 after retry RPC@1782660643), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1db0b3a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802) -> (continuation of Deferred@1612615514 after retry RPC@1782660643))) after lookup
12:08:46.782 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:46.783 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.783 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:46.784 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:47.661 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:47.661 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923),a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)[L],ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:47.662 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=11, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1249), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [7ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [7ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [48ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [49ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [51ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [51ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [109ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [366ms] refreshing cache from master, [366ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [368ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [370ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [372ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [372ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [1246ms] refreshing cache from master, [1246ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1248ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@731617576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@67c4c74e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802) -> (continuation of Deferred@1612615514 after retry RPC@1782660643) -> (continuation of Deferred@1512994468 after retry RPC@972942308), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1db0b3a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802) -> (continuation of Deferred@1612615514 after retry RPC@1782660643) -> (continuation of Deferred@1512994468 after retry RPC@972942308))) after lookup
12:08:47.662 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] connecting to peer
12:08:47.663 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:47.663 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] cleaning up while in state CONNECTING due to: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:47.664 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477) for tablet 2e0389dea5a34da1ba9cd502be7a4ee9: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477
12:08:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.727253  5678 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.727437  5678 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:47.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.728574  5678 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:47.729516  5382 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:47.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:47.730271  5382 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:47.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.733615  5615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165" is_pre_election: true
12:08:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.733870  5615 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 9.
12:08:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.734478  5382 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.734721  5678 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Leader pre-election won for term 11
12:08:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.734782  5678 raft_consensus.cc:493] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Starting leader election (detected failure of leader a535e5786752492ba8bd7288d7cc49d9)
12:08:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.734824  5678 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 10 FOLLOWER]: Advancing to term 11
12:08:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.735716  5678 raft_consensus.cc:515] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.736123  5678 leader_election.cc:290] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 11 election: Requested vote from peers ce8846e31f454232a6cf4bed41dcb165 (127.3.63.193:34755), a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.736500  5615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "2e0389dea5a34da1ba9cd502be7a4ee9" candidate_uuid: "7df98c593a294597b27ff79721619533" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "ce8846e31f454232a6cf4bed41dcb165"
12:08:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.736630  5615 raft_consensus.cc:3060] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 9 FOLLOWER]: Advancing to term 11
12:08:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:47.737079  5382 leader_election.cc:336] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111)
12:08:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.738433  5615 raft_consensus.cc:2468] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7df98c593a294597b27ff79721619533 in term 11.
12:08:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.738770  5382 leader_election.cc:304] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7df98c593a294597b27ff79721619533, ce8846e31f454232a6cf4bed41dcb165; no voters: a535e5786752492ba8bd7288d7cc49d9
12:08:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.738937  5678 raft_consensus.cc:2804] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 11 FOLLOWER]: Leader election won for term 11
12:08:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.739128  5678 raft_consensus.cc:697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [term 11 LEADER]: Becoming Leader. State: Replica: 7df98c593a294597b27ff79721619533, State: Running, Role: LEADER
12:08:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.739441  5678 consensus_queue.cc:237] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:47.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:47.742188  3513 catalog_manager.cc:5697] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 reported cstate change: term changed from 10 to 11, leader changed from a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195) to 7df98c593a294597b27ff79721619533 (127.3.63.194). New cstate: current_term: 11 leader_uuid: "7df98c593a294597b27ff79721619533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } health_report { overall_health: HEALTHY } } }
12:08:48.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.122200  5382 consensus_peers.cc:597] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 -> Peer a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477): Couldn't send request to peer a535e5786752492ba8bd7288d7cc49d9. Status: Network error: Client connection negotiation failed: client connection to 127.3.63.195:46477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:08:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.249121  5615 raft_consensus.cc:1275] T 2e0389dea5a34da1ba9cd502be7a4ee9 P ce8846e31f454232a6cf4bed41dcb165 [term 11 FOLLOWER]: Refusing update from remote peer 7df98c593a294597b27ff79721619533: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
12:08:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.249876  5678 consensus_queue.cc:1048] T 2e0389dea5a34da1ba9cd502be7a4ee9 P 7df98c593a294597b27ff79721619533 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
12:08:48.423 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 2e0389dea5a34da1ba9cd502be7a4ee9 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858' with partition [<start>, <end>)
12:08:48.424 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=2e0389dea5a34da1ba9cd502be7a4ee9@[7df98c593a294597b27ff79721619533(127.3.63.194:43923)[L],a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477),ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)]}]
12:08:48.424 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, attempt=13, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2012), Traces: [0ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [7ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [7ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [28ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [29ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [30ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [30ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [48ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [49ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [51ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [51ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [108ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [109ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [110ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [110ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [366ms] refreshing cache from master, [366ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [368ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [370ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [372ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [372ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [1246ms] refreshing cache from master, [1246ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [1248ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, [1250ms] sending RPC to server a535e5786752492ba8bd7288d7cc49d9, [1252ms] received response from server a535e5786752492ba8bd7288d7cc49d9: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [1252ms] delaying RPC due to: Network error: Failed to connect to peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477): Connection refused: /127.3.63.195:46477, [2006ms] refreshing cache from master, [2006ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.63.252:36877, [2011ms] Sub RPC GetTableLocations: received response from server master-127.3.63.252:36877: OK, deferred=Deferred@731617576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f189436b330@67c4c74e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802) -> (continuation of Deferred@1612615514 after retry RPC@1782660643) -> (continuation of Deferred@1512994468 after retry RPC@972942308) -> (continuation of Deferred@1332966316 after retry RPC@419894125), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1db0b3a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1122360017 after retry RPC@1292577952) -> (continuation of Deferred@359348403 after retry RPC@26310802) -> (continuation of Deferred@1612615514 after retry RPC@1782660643) -> (continuation of Deferred@1512994468 after retry RPC@972942308) -> (continuation of Deferred@1332966316 after retry RPC@419894125))) after lookup
12:08:48.425 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] connecting to peer
12:08:48.426 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] Successfully connected to peer
12:08:48.458 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x18e301d9, L:/127.0.0.1:54568 - R:/127.3.63.194:43923] using TOKEN/null
12:08:48.485 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.63.195:46477
12:08:48.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.486402  3327 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:08:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:08:48.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.608425  5709 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:08:48.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.608662  5709 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:08:48.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.608721  5709 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:08:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.612279  5709 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:08:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.612416  5709 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.63.195
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.616483  5709 tablet_server_runner.cc:78] Tablet server non-default flags:
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.63.212:38885
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data
12:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.63.195:46477
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.63.195
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45531
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.63.254:33189,127.3.63.253:39047,127.3.63.252:36877
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5709
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.63.195
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/logs
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on bdcb31816ec0
12:08:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16854
12:08:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.617908  5709 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:08:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.619058  5709 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:08:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.625649  5715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.625649  5714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:08:48.625710  5717 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:08:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.625986  5709 server_base.cc:1061] running on GCE node
12:08:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.626415  5709 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:08:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.626987  5709 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:08:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.628145  5709 hybrid_clock.cc:648] HybridClock initialized: now 1783512528628122 us; error 35 us; skew 500 ppm
12:08:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.630033  5709 webserver.cc:533] Webserver started at http://127.3.63.195:45531/ using document root <none> and password file <none>
12:08:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.630621  5709 fs_manager.cc:362] Metadata directory not provided
12:08:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.630681  5709 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:08:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.634234  5709 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
12:08:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.636426  5723 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:08:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.637539  5709 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.637722  5709 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data,/tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a535e5786752492ba8bd7288d7cc49d9"
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:08:18 on dist-test-slave-zwgr"
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.638173  5709 fs_report.cc:389] FS layout report
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/wal
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/data
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:08:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:08:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:08:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:08:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:08:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.659648  5709 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:08:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.660411  5709 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:08:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.660605  5709 kserver.cc:163] Server-wide thread pool size limit: 3276
12:08:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.661214  5709 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:08:48.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.662531  5731 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:08:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.665864  5709 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:08:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.665930  5709 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
12:08:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.665988  5709 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:08:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.667721  5709 ts_tablet_manager.cc:616] Registered 1 tablets
12:08:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.667773  5709 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.002s	sys 0.000s
12:08:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.668165  5731 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap starting.
12:08:48.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.678079  5709 rpc_server.cc:307] RPC server started. Bound to: 127.3.63.195:46477
12:08:48.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.678108  5843 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.63.195:46477 every 8 connection(s)
12:08:48.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.678992  5709 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDGNhPW/test-tmp/mini-kudu-cluster1502713675527926274/ts-2/data/info.pb
12:08:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.685635  5731 log.cc:826] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Log is configured to *not* fsync() on all Append() calls
12:08:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.686456  3327 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu as pid 5709
12:08:48.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:08:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.689955  5844 heartbeater.cc:344] Connected to a master server at 127.3.63.254:33189
12:08:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.690188  5844 heartbeater.cc:461] Registering TS with master...
12:08:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.690747  5844 heartbeater.cc:507] Master 127.3.63.254:33189 requested a full tablet report, sending...
12:08:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.692637  3368 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.694607  5845 heartbeater.cc:344] Connected to a master server at 127.3.63.252:36877
12:08:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.694705  5845 heartbeater.cc:461] Registering TS with master...
12:08:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.694924  5845 heartbeater.cc:507] Master 127.3.63.252:36877 requested a full tablet report, sending...
12:08:48.696 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.697034  3511 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.698002  3511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.63.195:33697
12:08:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.699589  5846 heartbeater.cc:344] Connected to a master server at 127.3.63.253:39047
12:08:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.699689  5846 heartbeater.cc:461] Registering TS with master...
12:08:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.699856  5846 heartbeater.cc:507] Master 127.3.63.253:39047 requested a full tablet report, sending...
12:08:48.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.700757  3437 ts_manager.cc:194] Re-registered known tserver with Master: a535e5786752492ba8bd7288d7cc49d9 (127.3.63.195:46477)
12:08:48.702 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512498858, tablet=2e0389dea5a34da1ba9cd502be7a4ee9, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:08:48.706 [DEBUG - main] (Connection.java:575) [peer master-127.3.63.254:33189(127.3.63.254:33189)] disconnecting while in state READY
12:08:48.707 [DEBUG - main] (Connection.java:575) [peer a535e5786752492ba8bd7288d7cc49d9(127.3.63.195:46477)] disconnecting while in state TERMINATED
12:08:48.707 [DEBUG - main] (Connection.java:575) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] disconnecting while in state READY
12:08:48.707 [DEBUG - main] (Connection.java:575) [peer master-127.3.63.253:39047(127.3.63.253:39047)] disconnecting while in state READY
12:08:48.709 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.3.63.253:39047(127.3.63.253:39047)] handling channelInactive
12:08:48.709 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.63.254:33189(127.3.63.254:33189)] handling channelInactive
12:08:48.710 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] handling channelInactive
12:08:48.710 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7df98c593a294597b27ff79721619533(127.3.63.194:43923)] cleaning up while in state READY due to: connection closed
12:08:48.710 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.63.253:39047(127.3.63.253:39047)] cleaning up while in state READY due to: connection closed
12:08:48.711 [DEBUG - main] (Connection.java:575) [peer ce8846e31f454232a6cf4bed41dcb165(127.3.63.193:34755)] disconnecting while in state TERMINATED
12:08:48.711 [DEBUG - main] (Connection.java:575) [peer master-127.3.63.252:36877(127.3.63.252:36877)] disconnecting while in state READY
12:08:48.711 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.3.63.252:36877(127.3.63.252:36877)] handling channelInactive
12:08:48.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.63.254:33189(127.3.63.254:33189)] cleaning up while in state READY due to: connection closed
12:08:48.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.63.252:36877(127.3.63.252:36877)] cleaning up while in state READY due to: connection closed
12:08:48.713 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
12:08:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.713982  5731 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:08:48.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.714429  5731 tablet_bootstrap.cc:492] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Bootstrap complete.
12:08:48.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.715346  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 5532
12:08:48.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.716214  5731 ts_tablet_manager.cc:1403] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent bootstrapping tablet: real 0.048s	user 0.033s	sys 0.011s
12:08:48.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.718881  5731 raft_consensus.cc:359] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:48.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.719308  5731 raft_consensus.cc:740] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: a535e5786752492ba8bd7288d7cc49d9, State: Initialized, Role: FOLLOWER
12:08:48.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.719813  5731 consensus_queue.cc:260] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce8846e31f454232a6cf4bed41dcb165" member_type: VOTER last_known_addr { host: "127.3.63.193" port: 34755 } } peers { permanent_uuid: "a535e5786752492ba8bd7288d7cc49d9" member_type: VOTER last_known_addr { host: "127.3.63.195" port: 46477 } } peers { permanent_uuid: "7df98c593a294597b27ff79721619533" member_type: VOTER last_known_addr { host: "127.3.63.194" port: 43923 } }
12:08:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.720496  5845 heartbeater.cc:499] Master 127.3.63.252:36877 was elected leader, sending a full tablet report...
12:08:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.720729  5731 ts_tablet_manager.cc:1434] T 2e0389dea5a34da1ba9cd502be7a4ee9 P a535e5786752492ba8bd7288d7cc49d9: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
12:08:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.725459  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 5364
12:08:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.731056  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 5709
12:08:48.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.739794  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 3338
12:08:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.745448  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 3407
12:08:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:08:48.750762  3327 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDGNhPW/build/debug/bin/kudu with pid 3479
12:08:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:08:48Z chronyd exiting

Time: 32.534

OK (1 test)

