JUnit version 4.13.2
.00:23:53.426 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783470233418
00:23:53.490 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
00:23:53.522 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:23:53.532 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4SKNut/test-tmp
00:23:53.560 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
00:23:53.564 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4SKNut/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
00:23:53.761 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

00:23:53.930 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:23:53.932 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

00:23:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:53.935185 24226 mini_oidc.cc:150] Starting JWKS server
00:23:53.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:53.937713 24226 webserver.cc:534] Webserver started at http://127.0.0.1:38781/ using document root <none> and password file <none>
00:23:53.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:53.938041 24226 mini_oidc.cc:186] Starting OIDC Discovery server
00:23:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:53.939167 24226 webserver.cc:534] Webserver started at http://127.0.0.1:37037/ using document root <none> and password file <none>
00:23:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:23:53Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
00:23:53.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:23:53Z Disabled control of system clock
00:23:53.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:53.967468 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:53.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:53.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal
00:23:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data
00:23:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:23:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:23:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:23:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/logs
00:23:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data/info.pb
00:23:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:23:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:23:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:23:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:23:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:23:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:23:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:23:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:23:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.190:39191
00:23:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.190
00:23:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:23:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:23:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38781/jwks.json with env {}
00:23:54.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.292798 24238 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:23:54.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.293111 24238 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:23:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.293195 24238 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:23:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.301739 24238 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:23:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.301854 24238 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:23:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.301913 24238 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:23:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.301986 24238 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:23:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.302096 24238 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:23:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.302136 24238 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38781/jwks.json
00:23:54.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.312467 24238 master_runner.cc:387] Master server non-default flags:
00:23:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:23:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.190:39191
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38781/jwks.json
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data/info.pb
00:23:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.190
00:23:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:23:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:23:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:23:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/logs
00:23:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:23:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:23:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:23:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:23:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:23:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:23:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:23:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:23:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:23:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.314381 24238 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:23:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.316270 24238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:23:54.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.324792 24244 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:54.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.324973 24243 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.326198 24238 server_base.cc:1061] running on GCE node
00:23:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.326879 24246 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.327782 24238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:23:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.329120 24238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:23:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.332422 24238 hybrid_clock.cc:648] HybridClock initialized: now 1783470234332060 us; error 349 us; skew 500 ppm
00:23:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.332744 24238 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:23:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.335068 24238 webserver.cc:534] Webserver started at http://127.23.168.190:33109/ using document root <none> and password file <none>
00:23:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.335765 24238 fs_manager.cc:362] Metadata directory not provided
00:23:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.335880 24238 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:23:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.336241 24238 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:23:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.338634 24238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data/instance:
00:23:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac1c677f0ac64ae188bc6f8ea64345ab"
00:23:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:54 on dist-test-slave-4ccz"
00:23:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.339442 24238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal/instance:
00:23:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac1c677f0ac64ae188bc6f8ea64345ab"
00:23:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:54 on dist-test-slave-4ccz"
00:23:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.344679 24238 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
00:23:54.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.348176 24252 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:23:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.349529 24238 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.002s
00:23:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.349663 24238 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal
00:23:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac1c677f0ac64ae188bc6f8ea64345ab"
00:23:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:54 on dist-test-slave-4ccz"
00:23:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.349871 24238 fs_report.cc:389] FS layout report
00:23:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:23:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal
00:23:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal
00:23:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data/data
00:23:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:23:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:23:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:23:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:23:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:23:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:23:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:23:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:23:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:23:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:23:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:23:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.378583 24238 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:23:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.379474 24238 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:23:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.379705 24238 kserver.cc:163] Server-wide thread pool size limit: 3276
00:23:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.400262 24238 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.190:39191
00:23:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.400286 24303 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.190:39191 every 8 connection(s)
00:23:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.401793 24238 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/data/info.pb
00:23:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.405092 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 24238
00:23:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.405330 24226 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-0/wal/instance
00:23:54.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.406237 24304 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:23:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.407562 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal
00:23:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data
00:23:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:23:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:23:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:23:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/logs
00:23:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data/info.pb
00:23:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:23:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:23:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:23:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:23:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:23:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:23:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:23:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.189:38643
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.189
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:23:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:23:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38781/jwks.json with env {}
00:23:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.414320 24304 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } has no permanent_uuid. Determining permanent_uuid...
00:23:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.435401 24304 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } has no permanent_uuid. Determining permanent_uuid...
00:23:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.436851 24255 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.189:38643: connect: Connection refused (error 111)
00:23:54.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.438696 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.189:38643: Network error: Client connection negotiation failed: client connection to 127.23.168.189:38643: connect: Connection refused (error 111)
00:23:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.488049 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } attempt: 1
00:23:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.490404 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.189:38643: Network error: Client connection negotiation failed: client connection to 127.23.168.189:38643: connect: Connection refused (error 111)
00:23:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.558658 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } attempt: 2
00:23:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.561247 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.189:38643: Network error: Client connection negotiation failed: client connection to 127.23.168.189:38643: connect: Connection refused (error 111)
00:23:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.652644 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } attempt: 3
00:23:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.658342 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.189:38643: Network error: Client connection negotiation failed: client connection to 127.23.168.189:38643: connect: Connection refused (error 111)
00:23:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.744897 24306 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:23:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.745311 24306 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:23:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.745438 24306 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:23:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.753926 24306 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:23:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.754094 24306 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:23:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.754180 24306 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:23:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.754240 24306 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:23:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.754323 24306 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:23:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.754374 24306 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38781/jwks.json
00:23:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.765100 24306 master_runner.cc:387] Master server non-default flags:
00:23:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:23:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.189:38643
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:23:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:23:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38781/jwks.json
00:23:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data/info.pb
00:23:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.189
00:23:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:23:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:23:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:23:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/logs
00:23:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:23:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:23:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:23:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:23:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:23:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:23:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:23:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:23:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:23:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.767428 24306 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:23:54.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.770056 24306 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:23:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.779021 24316 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:54.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.780643 24315 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:54.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.782498 24318 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:54.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.783434 24306 server_base.cc:1061] running on GCE node
00:23:54.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.784330 24306 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:23:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.786145 24306 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:23:54.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.787699 24306 hybrid_clock.cc:648] HybridClock initialized: now 1783470234787590 us; error 94 us; skew 500 ppm
00:23:54.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.788141 24306 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:23:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.790729 24306 webserver.cc:534] Webserver started at http://127.23.168.189:33523/ using document root <none> and password file <none>
00:23:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.791735 24306 fs_manager.cc:362] Metadata directory not provided
00:23:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.791839 24306 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:23:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.792234 24306 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:23:54.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.795353 24306 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data/instance:
00:23:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09da48273f3d4e179f7da69c602a4430"
00:23:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:54 on dist-test-slave-4ccz"
00:23:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.796303 24306 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal/instance:
00:23:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09da48273f3d4e179f7da69c602a4430"
00:23:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:54 on dist-test-slave-4ccz"
00:23:54.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.801628 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } attempt: 4
00:23:54.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.802513 24306 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.000s
00:23:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.803941 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.189:38643: Network error: Client connection negotiation failed: client connection to 127.23.168.189:38643: connect: Connection refused (error 111)
00:23:54.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.806394 24324 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.807850 24306 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.807998 24306 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09da48273f3d4e179f7da69c602a4430"
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:54 on dist-test-slave-4ccz"
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.808159 24306 fs_report.cc:389] FS layout report
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal
00:23:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data/data
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:23:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:23:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:23:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:23:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:23:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:23:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.828756 24306 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:23:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.829696 24306 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:23:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.829964 24306 kserver.cc:163] Server-wide thread pool size limit: 3276
00:23:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.850704 24306 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.189:38643
00:23:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.850723 24375 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.189:38643 every 8 connection(s)
00:23:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.852195 24306 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/data/info.pb
00:23:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.856145 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 24306
00:23:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.856273 24226 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-1/wal/instance
00:23:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.856678 24376 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:23:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.857168 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal
00:23:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/logs
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data/info.pb
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:23:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.188:44491
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.188
00:23:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:23:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38781/jwks.json with env {}
00:23:54.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.866878 24376 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } has no permanent_uuid. Determining permanent_uuid...
00:23:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.880103 24376 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } has no permanent_uuid. Determining permanent_uuid...
00:23:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.889257 24376 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } has no permanent_uuid. Determining permanent_uuid...
00:23:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.890583 24328 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.892179 24376 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:54.941496 24376 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 1
00:23:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:54.943622 24376 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.011922 24376 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 2
00:23:55.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.014144 24376 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.103204 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } attempt: 5
00:23:55.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.105391 24376 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 3
00:23:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.107585 24376 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.110872 24304 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } has no permanent_uuid. Determining permanent_uuid...
00:23:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.112847 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.164055 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 1
00:23:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.166116 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.197171 24378 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:23:55.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.197682 24378 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:23:55.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.197824 24378 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:23:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.207049 24378 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:23:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.207139 24378 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:23:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.207194 24378 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:23:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.207233 24378 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:23:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.207293 24378 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:23:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.207326 24378 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38781/jwks.json
00:23:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.217260 24378 master_runner.cc:387] Master server non-default flags:
00:23:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.188:44491
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:23:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38781/jwks.json
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data/info.pb
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.188
00:23:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/logs
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:23:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:23:55.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.219177 24378 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:23:55.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.221522 24378 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:23:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.230700 24387 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.230764 24386 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.231249 24389 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:55.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.232005 24378 server_base.cc:1061] running on GCE node
00:23:55.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.232838 24378 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:23:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.234429 24378 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:23:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.234349 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 2
00:23:55.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.235653 24378 hybrid_clock.cc:648] HybridClock initialized: now 1783470235235590 us; error 48 us; skew 500 ppm
00:23:55.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.236060 24378 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:23:55.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.236303 24304 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.238258 24378 webserver.cc:534] Webserver started at http://127.23.168.188:45629/ using document root <none> and password file <none>
00:23:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.238940 24378 fs_manager.cc:362] Metadata directory not provided
00:23:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.239025 24378 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:23:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.239323 24378 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:23:55.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.241549 24378 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data/instance:
00:23:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "065d65e936da4ec093b613bf0e9d3ae7"
00:23:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:55 on dist-test-slave-4ccz"
00:23:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.242312 24378 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal/instance:
00:23:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "065d65e936da4ec093b613bf0e9d3ae7"
00:23:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:55 on dist-test-slave-4ccz"
00:23:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.247370 24378 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.004s	sys 0.000s
00:23:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.250540 24395 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:23:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.250823 24376 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 4
00:23:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.252090 24378 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
00:23:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.252511 24378 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal
00:23:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "065d65e936da4ec093b613bf0e9d3ae7"
00:23:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:55 on dist-test-slave-4ccz"
00:23:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.252803 24376 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.168.188:44491: Network error: Client connection negotiation failed: client connection to 127.23.168.188:44491: connect: Connection refused (error 111)
00:23:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.253366 24378 fs_report.cc:389] FS layout report
00:23:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:23:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data/data
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:23:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:23:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:23:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:23:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:23:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.271941 24378 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:23:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.272794 24378 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:23:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.273025 24378 kserver.cc:163] Server-wide thread pool size limit: 3276
00:23:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.291998 24378 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.188:44491
00:23:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.292007 24446 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.188:44491 every 8 connection(s)
00:23:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.293433 24378 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/data/info.pb
00:23:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.296888 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 24378
00:23:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.297096 24226 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/master-2/wal/instance
00:23:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.297652 24447 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:23:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.305593 24447 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } has no permanent_uuid. Determining permanent_uuid...
00:23:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.318804 24447 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } has no permanent_uuid. Determining permanent_uuid...
00:23:55.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.327064 24447 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } has no permanent_uuid. Determining permanent_uuid...
00:23:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.336584 24447 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: Bootstrap starting.
00:23:55.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.339978 24447 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: Neither blocks nor log segments found. Creating new log.
00:23:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.340956 24447 log.cc:826] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: Log is configured to *not* fsync() on all Append() calls
00:23:55.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.342506 24304 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 3
00:23:55.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.343693 24447 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: No bootstrap required, opened a new log
00:23:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.351487 24447 raft_consensus.cc:359] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.351795 24447 raft_consensus.cc:385] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:23:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.351909 24447 raft_consensus.cc:740] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 065d65e936da4ec093b613bf0e9d3ae7, State: Initialized, Role: FOLLOWER
00:23:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.353248 24304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: Bootstrap starting.
00:23:55.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.353686 24447 consensus_queue.cc:260] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.354683 24454 sys_catalog.cc:455] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.355024 24454 sys_catalog.cc:458] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.355808 24447 sys_catalog.cc:565] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: configured and running, proceeding with master startup.
00:23:55.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.356858 24304 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: Neither blocks nor log segments found. Creating new log.
00:23:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.358071 24304 log.cc:826] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: Log is configured to *not* fsync() on all Append() calls
00:23:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.361268 24304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: No bootstrap required, opened a new log
00:23:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.366685 24304 raft_consensus.cc:359] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.367000 24304 raft_consensus.cc:385] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:23:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.367117 24304 raft_consensus.cc:740] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ac1c677f0ac64ae188bc6f8ea64345ab, State: Initialized, Role: FOLLOWER
00:23:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.367844 24304 consensus_queue.cc:260] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.372316 24466 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:23:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.372714 24466 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:23:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.373597 24467 sys_catalog.cc:455] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.374943 24304 sys_catalog.cc:565] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: configured and running, proceeding with master startup.
00:23:55.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.376263 24467 sys_catalog.cc:458] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.373688 24378 master_runner.cc:187] Error getting master registration for 127.23.168.190:39191: OK, instance_id { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" instance_seqno: 1783470234394773 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:23:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.380685 24378 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:23:55.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.382215 24258 tablet.cc:2406] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:23:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.391467 24479 catalog_manager.cc:1593] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:23:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.393581 24479 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:23:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.390597 24238 master_runner.cc:187] Error getting master registration for 127.23.168.189:38643: OK, instance_id { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" instance_seqno: 1783470234845210 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:23:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.397619 24238 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:23:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.525097 24401 tablet.cc:2406] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:23:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.536260 24467 raft_consensus.cc:493] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:23:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.536634 24467 raft_consensus.cc:515] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.538656 24467 leader_election.cc:290] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 09da48273f3d4e179f7da69c602a4430 (127.23.168.189:38643), 065d65e936da4ec093b613bf0e9d3ae7 (127.23.168.188:44491)
00:23:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.539842 24422 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "065d65e936da4ec093b613bf0e9d3ae7" is_pre_election: true
00:23:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.540447 24422 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ac1c677f0ac64ae188bc6f8ea64345ab in term 0.
00:23:55.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.541132 24256 leader_election.cc:304] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 065d65e936da4ec093b613bf0e9d3ae7, ac1c677f0ac64ae188bc6f8ea64345ab; no voters: 
00:23:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.542282 24467 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Leader pre-election won for term 1
00:23:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.542434 24467 raft_consensus.cc:493] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:23:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.542550 24467 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 0 FOLLOWER]: Advancing to term 1
00:23:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.545845 24467 raft_consensus.cc:515] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.547128 24422 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "065d65e936da4ec093b613bf0e9d3ae7"
00:23:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.547331 24422 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 0 FOLLOWER]: Advancing to term 1
00:23:55.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.550978 24422 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ac1c677f0ac64ae188bc6f8ea64345ab in term 1.
00:23:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.551467 24256 leader_election.cc:304] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 065d65e936da4ec093b613bf0e9d3ae7, ac1c677f0ac64ae188bc6f8ea64345ab; no voters: 
00:23:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.552002 24376 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } attempt: 5
00:23:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.552686 24481 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 1 FOLLOWER]: Leader election won for term 1
00:23:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.566663 24376 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: Bootstrap starting.
00:23:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.560940 24467 leader_election.cc:290] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [CANDIDATE]: Term 1 election: Requested vote from peers 09da48273f3d4e179f7da69c602a4430 (127.23.168.189:38643), 065d65e936da4ec093b613bf0e9d3ae7 (127.23.168.188:44491)
00:23:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.571290 24376 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: Neither blocks nor log segments found. Creating new log.
00:23:55.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.574164 24376 log.cc:826] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: Log is configured to *not* fsync() on all Append() calls
00:23:55.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.579487 24481 raft_consensus.cc:697] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [term 1 LEADER]: Becoming Leader. State: Replica: ac1c677f0ac64ae188bc6f8ea64345ab, State: Running, Role: LEADER
00:23:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.581579 24376 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: No bootstrap required, opened a new log
00:23:55.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.581389 24481 consensus_queue.cc:237] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.589840 24376 raft_consensus.cc:359] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.590517 24376 raft_consensus.cc:385] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:23:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.590950 24376 raft_consensus.cc:740] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 09da48273f3d4e179f7da69c602a4430, State: Initialized, Role: FOLLOWER
00:23:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.591946 24376 consensus_queue.cc:260] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } }
00:23:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.598145 24467 sys_catalog.cc:455] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: SysCatalogTable state changed. Reason: New leader ac1c677f0ac64ae188bc6f8ea64345ab. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.598425 24467 sys_catalog.cc:458] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: This master's current role is: LEADER
00:23:55.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.607749 24486 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
00:23:55.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.609843 24376 sys_catalog.cc:565] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: configured and running, proceeding with master startup.
00:23:55.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.607883 24485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.612219 24485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.612933 24351 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "09da48273f3d4e179f7da69c602a4430" is_pre_election: true
00:23:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.613521 24351 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ac1c677f0ac64ae188bc6f8ea64345ab in term 0.
00:23:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.614969 24350 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "09da48273f3d4e179f7da69c602a4430"
00:23:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.615250 24350 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 0 FOLLOWER]: Advancing to term 1
00:23:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.618875 24350 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ac1c677f0ac64ae188bc6f8ea64345ab in term 1.
00:23:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.628551 24486 catalog_manager.cc:1519] Initializing Kudu cluster ID...
00:23:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.658991 24350 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [term 1 FOLLOWER]: Refusing update from remote peer ac1c677f0ac64ae188bc6f8ea64345ab: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:23:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.659076 24421 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [term 1 FOLLOWER]: Refusing update from remote peer ac1c677f0ac64ae188bc6f8ea64345ab: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:23:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.660444 24481 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [LEADER]: Connected to new peer: Peer: permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:23:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.661721 24481 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [LEADER]: Connected to new peer: Peer: permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:23:55.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.687877 24485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ac1c677f0ac64ae188bc6f8ea64345ab. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.688170 24485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.690239 24454 sys_catalog.cc:455] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ac1c677f0ac64ae188bc6f8ea64345ab. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.690506 24454 sys_catalog.cc:458] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.700119 24496 mvcc.cc:204] Tried to move back new op lower bound from 7305094085248454656 to 7305094084958334976. Current Snapshot: MvccSnapshot[applied={T|T < 7305094085248454656}]
00:23:55.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.705351 24454 sys_catalog.cc:455] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.705579 24454 sys_catalog.cc:458] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7 [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.710156 24495 mvcc.cc:204] Tried to move back new op lower bound from 7305094085248454656 to 7305094084958334976. Current Snapshot: MvccSnapshot[applied={T|T < 7305094085248454656}]
00:23:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.721601 24485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.721874 24485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430 [sys.catalog]: This master's current role is: FOLLOWER
00:23:55.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.727108 24486 catalog_manager.cc:1382] Generated new cluster ID: 8fa47e6bb5bf4e7c852f24d92b131ebe
00:23:55.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.727236 24486 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
00:23:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.739250 24467 sys_catalog.cc:455] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.739506 24467 sys_catalog.cc:458] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: This master's current role is: LEADER
00:23:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.740384 24467 sys_catalog.cc:455] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac1c677f0ac64ae188bc6f8ea64345ab" member_type: VOTER last_known_addr { host: "127.23.168.190" port: 39191 } } peers { permanent_uuid: "09da48273f3d4e179f7da69c602a4430" member_type: VOTER last_known_addr { host: "127.23.168.189" port: 38643 } } peers { permanent_uuid: "065d65e936da4ec093b613bf0e9d3ae7" member_type: VOTER last_known_addr { host: "127.23.168.188" port: 44491 } } }
00:23:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.740589 24467 sys_catalog.cc:458] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab [sys.catalog]: This master's current role is: LEADER
00:23:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.758543 24486 catalog_manager.cc:1405] Generated new certificate authority record
00:23:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.759470 24505 catalog_manager.cc:1294] Loaded cluster ID: 8fa47e6bb5bf4e7c852f24d92b131ebe
00:23:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.759907 24505 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: loading cluster ID for follower catalog manager: success
00:23:55.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.760890 24486 catalog_manager.cc:1539] Loading token signing keys...
00:23:55.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.761583 24505 catalog_manager.cc:1611] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: acquiring CA information for follower catalog manager: Not found: root CA entry not found
00:23:55.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:55.761917 24505 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
00:23:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.785041 24486 catalog_manager.cc:6069] T 00000000000000000000000000000000 P ac1c677f0ac64ae188bc6f8ea64345ab: Generated new TSK 0
00:23:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.786703 24486 catalog_manager.cc:1549] Initializing in-progress tserver states...
00:23:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:55.804237 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:0
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:23:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:23:56.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.234887 24507 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:23:56.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.235255 24507 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:23:56.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.235374 24507 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:23:56.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.244019 24507 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:23:56.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.244242 24507 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.129
00:23:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.254592 24507 tablet_server_runner.cc:78] Tablet server non-default flags:
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:23:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:0
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:23:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:23:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:23:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.256896 24507 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:23:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.259204 24507 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:23:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.269280 24512 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.269829 24513 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:56.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.271760 24515 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:56.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.271787 24507 server_base.cc:1061] running on GCE node
00:23:56.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.272521 24507 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:23:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.273689 24507 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:23:56.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.274953 24507 hybrid_clock.cc:648] HybridClock initialized: now 1783470236274768 us; error 166 us; skew 500 ppm
00:23:56.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.275365 24507 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:23:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.278405 24507 webserver.cc:534] Webserver started at http://127.23.168.129:32839/ using document root <none> and password file <none>
00:23:56.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.279425 24507 fs_manager.cc:362] Metadata directory not provided
00:23:56.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.279554 24507 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.280002 24507 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:23:56.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.283413 24507 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/instance:
00:23:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:23:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:23:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.284469 24507 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal/instance:
00:23:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:23:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:23:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.291756 24507 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.003s
00:23:56.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.296957 24521 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:23:56.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.298694 24507 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.000s
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.298918 24507 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.299178 24507 fs_report.cc:389] FS layout report
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/data
00:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:23:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:23:56.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.342900 24507 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:23:56.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.344036 24507 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:23:56.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.344384 24507 kserver.cc:163] Server-wide thread pool size limit: 3276
00:23:56.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.345537 24507 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:23:56.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.347611 24507 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:23:56.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.347759 24507 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:23:56.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.347872 24507 ts_tablet_manager.cc:616] Registered 0 tablets
00:23:56.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.347954 24507 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:23:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.375450 24466 catalog_manager.cc:1294] Loaded cluster ID: 8fa47e6bb5bf4e7c852f24d92b131ebe
00:23:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.375577 24466 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: loading cluster ID for follower catalog manager: success
00:23:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.378975 24466 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: acquiring CA information for follower catalog manager: success
00:23:56.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.381762 24466 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 065d65e936da4ec093b613bf0e9d3ae7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:23:56.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.387547 24507 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.129:40013
00:23:56.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.387621 24633 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.129:40013 every 8 connection(s)
00:23:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.388936 24507 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:23:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.395148 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 24507
00:23:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.395409 24226 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal/instance
00:23:56.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.401710 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:23:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:0
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:23:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:23:56.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.423189 24635 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:23:56.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.424125 24635 heartbeater.cc:461] Registering TS with master...
00:23:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.425660 24635 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:23:56.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.427810 24634 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:23:56.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.427942 24634 heartbeater.cc:461] Registering TS with master...
00:23:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.428440 24634 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:23:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.428774 24636 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:23:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.428898 24636 heartbeater.cc:461] Registering TS with master...
00:23:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.428893 24269 ts_manager.cc:194] Registered new tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:23:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.429409 24636 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:23:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.431033 24269 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.129:33163
00:23:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.432032 24341 ts_manager.cc:194] Registered new tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:23:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.434844 24412 ts_manager.cc:194] Registered new tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:23:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.761040 24644 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:23:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.761363 24644 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:23:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.761444 24644 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:23:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.765594 24505 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: acquiring CA information for follower catalog manager: success
00:23:56.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.767545 24505 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 09da48273f3d4e179f7da69c602a4430: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:23:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.770175 24644 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:23:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.770402 24644 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.130
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.781240 24644 tablet_server_runner.cc:78] Tablet server non-default flags:
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:23:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:0
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:23:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:23:56.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.783241 24644 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.785169 24644 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:23:56.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.795331 24649 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.795435 24650 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:56.796082 24652 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.796788 24644 server_base.cc:1061] running on GCE node
00:23:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.797542 24644 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:23:56.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.798816 24644 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:23:56.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.800275 24644 hybrid_clock.cc:648] HybridClock initialized: now 1783470236800203 us; error 52 us; skew 500 ppm
00:23:56.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.800690 24644 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:23:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.803598 24644 webserver.cc:534] Webserver started at http://127.23.168.130:36067/ using document root <none> and password file <none>
00:23:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.804306 24644 fs_manager.cc:362] Metadata directory not provided
00:23:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.804430 24644 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:23:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.804736 24644 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:23:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.807278 24644 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/instance:
00:23:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:23:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:23:56.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.808149 24644 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal/instance:
00:23:56.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:23:56.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:23:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.814015 24644 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.001s
00:23:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.817588 24658 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.819115 24644 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.819300 24644 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.819499 24644 fs_report.cc:389] FS layout report
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/data
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:23:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:23:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:23:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:23:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:23:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:23:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:23:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.836048 24644 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:23:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.836921 24644 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:23:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.837160 24644 kserver.cc:163] Server-wide thread pool size limit: 3276
00:23:56.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.838186 24644 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:23:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.839893 24644 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:23:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.839991 24644 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:23:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.840063 24644 ts_tablet_manager.cc:616] Registered 0 tablets
00:23:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.840109 24644 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:23:56.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.876804 24644 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.130:34095
00:23:56.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.876838 24770 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.130:34095 every 8 connection(s)
00:23:56.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.878326 24644 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:23:56.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.884670 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 24644
00:23:56.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.884850 24226 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal/instance
00:23:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.889256 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:23:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:0
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:23:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.897531 24772 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:23:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.897616 24771 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:23:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.897557 24773 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:23:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.898062 24773 heartbeater.cc:461] Registering TS with master...
00:23:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.898056 24772 heartbeater.cc:461] Registering TS with master...
00:23:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.898051 24771 heartbeater.cc:461] Registering TS with master...
00:23:56.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.899371 24772 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:23:56.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.899400 24773 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:23:56.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.899596 24771 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:23:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.901185 24269 ts_manager.cc:194] Registered new tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:23:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.901352 24341 ts_manager.cc:194] Registered new tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:23:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.902371 24269 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.130:55473
00:23:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:56.902669 24412 ts_manager.cc:194] Registered new tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:23:57.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.248726 24778 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:23:57.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.249028 24778 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:23:57.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.249107 24778 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:23:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.257326 24778 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:23:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.257500 24778 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.131
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.267803 24778 tablet_server_runner.cc:78] Tablet server non-default flags:
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:0
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:23:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:23:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.269658 24778 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:23:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.271520 24778 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:23:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.281451 24783 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.281447 24786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:57.281404 24784 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:23:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.282272 24778 server_base.cc:1061] running on GCE node
00:23:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.282902 24778 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:23:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.283912 24778 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:23:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.285151 24778 hybrid_clock.cc:648] HybridClock initialized: now 1783470237285100 us; error 36 us; skew 500 ppm
00:23:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.285506 24778 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:23:57.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.287709 24778 webserver.cc:534] Webserver started at http://127.23.168.131:32819/ using document root <none> and password file <none>
00:23:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.288496 24778 fs_manager.cc:362] Metadata directory not provided
00:23:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.288631 24778 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:23:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.289028 24778 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:23:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.291615 24778 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/instance:
00:23:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:23:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:57 on dist-test-slave-4ccz"
00:23:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.292618 24778 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal/instance:
00:23:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:23:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:57 on dist-test-slave-4ccz"
00:23:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.299543 24778 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.003s	sys 0.005s
00:23:57.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.303977 24792 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:23:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.305576 24778 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
00:23:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.305728 24778 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:23:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:57 on dist-test-slave-4ccz"
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.305897 24778 fs_report.cc:389] FS layout report
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/data
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:23:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:23:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.320154 24778 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:23:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.320987 24778 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:23:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.321220 24778 kserver.cc:163] Server-wide thread pool size limit: 3276
00:23:57.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.322265 24778 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:23:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.324016 24778 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:23:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.324101 24778 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:23:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.324172 24778 ts_tablet_manager.cc:616] Registered 0 tablets
00:23:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.324229 24778 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:23:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.359745 24778 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.131:37089
00:23:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.359820 24904 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.131:37089 every 8 connection(s)
00:23:57.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.361284 24778 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:23:57.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.371300 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 24778
00:23:57.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.371498 24226 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal/instance
00:23:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.379416 24907 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:23:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.379876 24907 heartbeater.cc:461] Registering TS with master...
00:23:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.380666 24905 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:23:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.380792 24905 heartbeater.cc:461] Registering TS with master...
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.381194 24907 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.381199 24905 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.381603 24906 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.381724 24906 heartbeater.cc:461] Registering TS with master...
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.382227 24906 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.383181 24268 ts_manager.cc:194] Registered new tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:23:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.383347 24341 ts_manager.cc:194] Registered new tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:23:57.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.384093 24412 ts_manager.cc:194] Registered new tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:23:57.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.384133 24268 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.131:51821
00:23:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.399041 24226 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
00:23:57.401 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:23:57.402 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

00:23:57.413 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.23.168.190"
      port: 39191
    }
    bound_http_address {
      host: "127.23.168.190"
      port: 33109
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.23.168.189"
      port: 38643
    }
    bound_http_address {
      host: "127.23.168.189"
      port: 33523
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.23.168.188"
      port: 44491
    }
    bound_http_address {
      host: "127.23.168.188"
      port: 45629
    }
  }
}

00:23:57.416 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

00:23:57.419 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.23.168.129"
      port: 40013
    }
    bound_http_address {
      host: "127.23.168.129"
      port: 32839
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.23.168.130"
      port: 34095
    }
    bound_http_address {
      host: "127.23.168.130"
      port: 36067
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.23.168.131"
      port: 37089
    }
    bound_http_address {
      host: "127.23.168.131"
      port: 32819
    }
  }
}

00:23:57.419 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:23:57.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.434710 24635 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:23:57.581 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
00:23:57.648 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
00:23:57.831 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.168.189:38643(127.23.168.189:38643)] connecting to peer
00:23:57.894 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.168.188:44491(127.23.168.188:44491)] connecting to peer
00:23:57.895 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.168.190:39191(127.23.168.190:39191)] connecting to peer
00:23:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:57.904727 24772 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:23:57.921 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.168.188:44491(127.23.168.188:44491)] Successfully connected to peer
00:23:57.922 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.168.189:38643(127.23.168.189:38643)] Successfully connected to peer
00:23:57.922 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.23.168.190:39191(127.23.168.190:39191)] Successfully connected to peer
00:23:58.017 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.168.189
00:23:58.019 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.168.188
00:23:58.025 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.168.190
00:23:58.229 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xdb85517e, L:/127.0.0.1:49674 - R:/127.23.168.188:44491] using SASL/PLAIN
00:23:58.229 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x0c6892a5, L:/127.0.0.1:39302 - R:/127.23.168.190:39191] using SASL/PLAIN
00:23:58.229 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8d09c3b3, L:/127.0.0.1:33314 - R:/127.23.168.189:38643] using SASL/PLAIN
00:23:58.264 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
00:23:58.265 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=472), Traces: [0ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.168.189:38643, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.168.188:44491, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.168.190:39191, [446ms] Sub RPC ConnectToMaster: received response from server master-127.23.168.188:44491: OK, [446ms] Sub RPC ConnectToMaster: received response from server master-127.23.168.189:38643: OK, [447ms] Sub RPC ConnectToMaster: received response from server master-127.23.168.190:39191: OK, deferred=null) after lookup
00:23:58.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.313202 24268 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39302:
00:23:58.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778"
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:23:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:23:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
00:23:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
00:23:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
00:23:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
00:23:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:23:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:23:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:58.315496 24268 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
00:23:58.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.386379 24906 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:23:58.430 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 6eaa7afbbe79466c8292d3a26b66cb4b
00:23:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.429992 24569 tablet_service.cc:1511] Processing CreateTablet for tablet c17cfea4b88143bc98ad44ebf3f798ac (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778 [id=6eaa7afbbe79466c8292d3a26b66cb4b]), partition=RANGE (key) PARTITION UNBOUNDED
00:23:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.430017 24706 tablet_service.cc:1511] Processing CreateTablet for tablet c17cfea4b88143bc98ad44ebf3f798ac (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778 [id=6eaa7afbbe79466c8292d3a26b66cb4b]), partition=RANGE (key) PARTITION UNBOUNDED
00:23:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.431607 24706 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c17cfea4b88143bc98ad44ebf3f798ac. 1 dirs total, 0 dirs full, 0 dirs failed
00:23:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.431634 24569 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c17cfea4b88143bc98ad44ebf3f798ac. 1 dirs total, 0 dirs full, 0 dirs failed
00:23:58.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.441028 24840 tablet_service.cc:1511] Processing CreateTablet for tablet c17cfea4b88143bc98ad44ebf3f798ac (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778 [id=6eaa7afbbe79466c8292d3a26b66cb4b]), partition=RANGE (key) PARTITION UNBOUNDED
00:23:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.442580 24840 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c17cfea4b88143bc98ad44ebf3f798ac. 1 dirs total, 0 dirs full, 0 dirs failed
00:23:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.445309 24938 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap starting.
00:23:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.450613 24938 tablet_bootstrap.cc:654] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Neither blocks nor log segments found. Creating new log.
00:23:58.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.452291 24938 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log is configured to *not* fsync() on all Append() calls
00:23:58.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.456090 24940 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap starting.
00:23:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.457696 24941 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap starting.
00:23:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.457882 24938 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: No bootstrap required, opened a new log
00:23:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.458209 24938 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent bootstrapping tablet: real 0.013s	user 0.007s	sys 0.004s
00:23:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.460556 24940 tablet_bootstrap.cc:654] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Neither blocks nor log segments found. Creating new log.
00:23:58.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.461714 24940 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Log is configured to *not* fsync() on all Append() calls
00:23:58.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.461882 24941 tablet_bootstrap.cc:654] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Neither blocks nor log segments found. Creating new log.
00:23:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.463158 24941 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Log is configured to *not* fsync() on all Append() calls
00:23:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.464648 24940 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: No bootstrap required, opened a new log
00:23:58.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.464959 24940 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent bootstrapping tablet: real 0.009s	user 0.006s	sys 0.000s
00:23:58.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.466005 24941 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: No bootstrap required, opened a new log
00:23:58.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.466269 24941 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent bootstrapping tablet: real 0.009s	user 0.008s	sys 0.000s
00:23:58.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.466147 24938 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.466547 24938 raft_consensus.cc:385] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:23:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.466861 24938 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Initialized, Role: FOLLOWER
00:23:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.472216 24940 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.472832 24940 raft_consensus.cc:385] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:23:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.473083 24940 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Initialized, Role: FOLLOWER
00:23:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.473544 24941 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.473922 24940 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.476392 24941 raft_consensus.cc:385] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:23:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.476670 24941 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Initialized, Role: FOLLOWER
00:23:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.467640 24938 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.479046 24941 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.479812 24938 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent starting tablet: real 0.021s	user 0.008s	sys 0.004s
00:23:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.491313 24940 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent starting tablet: real 0.026s	user 0.014s	sys 0.012s
00:23:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.493160 24941 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent starting tablet: real 0.027s	user 0.013s	sys 0.005s
00:23:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.493204 24944 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:23:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.493431 24944 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.495873 24944 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:23:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.508919 24726 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" is_pre_election: true
00:23:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.509537 24726 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 0.
00:23:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.510318 24525 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 
00:23:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.510890 24944 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:23:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.511057 24944 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:23:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.510891 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:23:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.511150 24944 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 0 FOLLOWER]: Advancing to term 1
00:23:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.511498 24860 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 0.
00:23:58.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.513820 24944 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.514525 24944 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 1 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:23:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.515000 24726 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:23:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.515017 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:23:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.515202 24860 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 0 FOLLOWER]: Advancing to term 1
00:23:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.515205 24726 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 0 FOLLOWER]: Advancing to term 1
00:23:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.517800 24726 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 1.
00:23:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.518225 24525 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 
00:23:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.518250 24860 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 1.
00:23:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.518503 24944 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Leader election won for term 1
00:23:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.519815 24944 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 LEADER]: Becoming Leader. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Running, Role: LEADER
00:23:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.520373 24944 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:23:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.526095 24268 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 reported cstate change: term changed from 0 to 1, leader changed from <none> to af84f760f77249aa81d9789e1b93e381 (127.23.168.129). New cstate: current_term: 1 leader_uuid: "af84f760f77249aa81d9789e1b93e381" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:23:58.541 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 6eaa7afbbe79466c8292d3a26b66cb4b
00:23:58.560 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:58.565 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:58.566 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [8ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=null) after lookup
00:23:58.581 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:58.583 [DEBUG - kudu-nio-3] (Connection.java:757) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] Successfully connected to peer
00:23:58.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:58.615140 24908 tablet.cc:2406] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:23:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:58.632422 24774 tablet.cc:2406] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:23:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:23:58.644521 24637 tablet.cc:2406] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:23:58.645 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xf2d10c60, L:/127.0.0.1:59748 - R:/127.23.168.129:40013] using TOKEN/null
00:23:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.655340 24860 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Refusing update from remote peer af84f760f77249aa81d9789e1b93e381: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:23:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.655340 24726 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Refusing update from remote peer af84f760f77249aa81d9789e1b93e381: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:23:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.656661 24949 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:23:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.657015 24944 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:23:58.780 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:23:58.781 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.129:40013
00:23:58.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:23:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.783967 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 24507
00:23:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.794258 24629 generic_service.cc:196] Checking for leaks (request via RPC)
00:23:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:23:58.907398 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 24507
00:23:58.928 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:23:58.930 [DEBUG - kudu-nio-3] (Connection.java:239) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] handling channelInactive
00:23:58.931 [DEBUG - kudu-nio-3] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state READY due to: connection closed
00:23:58.931 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: connection closed
00:23:58.938 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:58.938 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:58.939 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test)) after lookup
00:23:58.939 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:58.942 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:58.942 [DEBUG - kudu-nio-4] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:58.942 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:58.958 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:58.958 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:58.959 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [11ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407))) after lookup
00:23:58.960 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:58.962 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:58.962 [DEBUG - kudu-nio-5] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:58.963 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:58.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:58.999 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:59.000 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [11ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [32ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [70ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416))) after lookup
00:23:59.000 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:59.002 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.003 [DEBUG - kudu-nio-6] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.003 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.058 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:59.059 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:59.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=131), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [11ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [32ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [70ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [75ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [75ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [130ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010))) after lookup
00:23:59.060 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:59.062 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.062 [DEBUG - kudu-nio-7] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.062 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.118 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:59.119 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:59.120 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=191), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [11ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [32ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [70ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [75ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [75ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [130ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [131ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [134ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [134ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [186ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [190ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010) -> (continuation of Deferred@11622242 after retry RPC@2026225126), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010) -> (continuation of Deferred@11622242 after retry RPC@2026225126))) after lookup
00:23:59.120 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:59.122 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.122 [DEBUG - kudu-nio-0] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.122 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.758 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:23:59.758 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:23:59.759 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=830), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [11ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [32ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [70ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [75ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [75ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [130ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [131ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [134ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [134ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [186ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [190ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [192ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [194ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [194ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [826ms] refreshing cache from master, [826ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [829ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010) -> (continuation of Deferred@11622242 after retry RPC@2026225126) -> (continuation of Deferred@1505129968 after retry RPC@455755911), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010) -> (continuation of Deferred@11622242 after retry RPC@2026225126) -> (continuation of Deferred@1505129968 after retry RPC@455755911))) after lookup
00:23:59.760 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:23:59.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:23:59.761 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:00.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.232252 24968 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:00.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.232465 24968 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:00.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.234234 24969 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:00.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.234634 24969 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.239588 24969 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.240592 24968 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:00.241971 24795 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:00.242723 24661 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:00.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:00.244521 24795 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:00.245519 24661 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:00.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.248051 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:00.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.248275 24860 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 1.
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.248802 24660 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.248867 24726 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" is_pre_election: true
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.249112 24726 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 1.
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.249385 24969 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Leader pre-election won for term 2
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.249534 24969 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Starting leader election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.249627 24969 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 1 FOLLOWER]: Advancing to term 2
00:24:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.249675 24796 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:00.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.250108 24968 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Leader pre-election won for term 2
00:24:00.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.250241 24968 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Starting leader election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:00.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.250432 24968 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 1 FOLLOWER]: Advancing to term 2
00:24:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.252173 24969 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.252645 24968 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.252774 24969 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 2 election: Requested vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.253402 24968 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 2 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.253378 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.253715 24860 raft_consensus.cc:2393] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in current term 2: Already voted for candidate 81d9cd709bc146289a569ff26e47b3c8 in this term.
00:24:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.254181 24726 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.254439 24726 raft_consensus.cc:2393] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 81d9cd709bc146289a569ff26e47b3c8 in current term 2: Already voted for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in this term.
00:24:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:00.255405 24661 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.255537 24661 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381
00:24:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.255873 24969 raft_consensus.cc:2749] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:24:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:00.256168 24795 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.256309 24795 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381
00:24:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:00.256588 24968 raft_consensus.cc:2749] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:24:01.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.889945 24978 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:01.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.890165 24978 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:01.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.890781 24978 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:01.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.891454 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:01.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.891693 24860 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 2.
00:24:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.892254 24660 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: 
00:24:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.892583 24978 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Leader pre-election won for term 3
00:24:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.892721 24978 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:24:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.892828 24978 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 2 FOLLOWER]: Advancing to term 3
00:24:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:01.893168 24661 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:01.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.895058 24978 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:01.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.895692 24978 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 3 election: Requested vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.896240 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.896440 24860 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 2 FOLLOWER]: Advancing to term 3
00:24:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:01.896978 24661 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.898835 24860 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 3.
00:24:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.899289 24660 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.899567 24978 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 FOLLOWER]: Leader election won for term 3
00:24:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.899958 24978 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 LEADER]: Becoming Leader. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Running, Role: LEADER
00:24:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.900521 24978 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:01.904552 24268 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 reported cstate change: term changed from 1 to 3, leader changed from af84f760f77249aa81d9789e1b93e381 (127.23.168.129) to 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130). New cstate: current_term: 3 leader_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:02.329735 24860 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Refusing update from remote peer 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
00:24:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:02.330500 24978 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
00:24:02.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:02.340726 24661 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 -> Peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Couldn't send request to peer af84f760f77249aa81d9789e1b93e381. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:03.558 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:03.558 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:03.559 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=4630), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [3ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [11ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [32ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [70ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [75ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [75ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [130ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [131ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [134ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [134ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [186ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [190ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [192ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [194ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [194ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [826ms] refreshing cache from master, [826ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [829ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [831ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [833ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [833ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [4626ms] refreshing cache from master, [4626ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [4629ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1320485127(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@2aad6c80 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010) -> (continuation of Deferred@11622242 after retry RPC@2026225126) -> (continuation of Deferred@1505129968 after retry RPC@455755911) -> (continuation of Deferred@2034171252 after retry RPC@807005348), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@573d69e4 -> wakeup thread Time-limited test -> (continuation of Deferred@651290502 after retry RPC@2094286407) -> (continuation of Deferred@265314248 after retry RPC@2005083416) -> (continuation of Deferred@470000553 after retry RPC@1398978010) -> (continuation of Deferred@11622242 after retry RPC@2026225126) -> (continuation of Deferred@1505129968 after retry RPC@455755911) -> (continuation of Deferred@2034171252 after retry RPC@807005348))) after lookup
00:24:03.560 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:03.561 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] Successfully connected to peer
00:24:03.581 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x3bd0c5e8, L:/127.0.0.1:37226 - R:/127.23.168.130:34095] using TOKEN/null
00:24:03.605 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.129:40013
00:24:03.606 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.607980 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.948648 25009 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.948958 25009 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.949043 25009 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.957468 25009 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.957638 25009 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.129
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.967950 25009 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.969730 25009 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:03.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.971532 25009 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.982165 25017 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.982297 25019 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:03.982113 25016 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:03.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.983742 25009 server_base.cc:1061] running on GCE node
00:24:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.984304 25009 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.985249 25009 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.986469 25009 hybrid_clock.cc:648] HybridClock initialized: now 1783470243986404 us; error 54 us; skew 500 ppm
00:24:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.986804 25009 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.988993 25009 webserver.cc:534] Webserver started at http://127.23.168.129:32839/ using document root <none> and password file <none>
00:24:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.989678 25009 fs_manager.cc:362] Metadata directory not provided
00:24:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.989778 25009 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.995347 25009 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
00:24:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.998286 25025 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:03.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.999794 25009 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:03.999974 25009 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.000716 25009 fs_report.cc:389] FS layout report
00:24:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/data
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.020788 25009 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.021726 25009 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:04.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.022028 25009 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.023037 25009 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:04.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.025630 25032 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:04.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.031857 25009 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:04.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.031960 25009 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:24:04.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.032069 25009 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.035293 25009 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.035393 25009 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.002s	sys 0.000s
00:24:04.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.035951 25032 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap starting.
00:24:04.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.058461 25032 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Log is configured to *not* fsync() on all Append() calls
00:24:04.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.075984 25009 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.129:40013
00:24:04.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.076045 25139 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.129:40013 every 8 connection(s)
00:24:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.077515 25032 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.077797 25009 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.078297 25032 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap complete.
00:24:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.080281 25032 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent bootstrapping tablet: real 0.045s	user 0.037s	sys 0.004s
00:24:04.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.085994 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 25009
00:24:04.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:04.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.092892 25032 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.093906 25032 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Initialized, Role: FOLLOWER
00:24:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.094821 25032 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:04.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.110787 25032 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent starting tablet: real 0.030s	user 0.011s	sys 0.008s
00:24:04.114 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:04.114 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.130:34095
00:24:04.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:24:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.115549 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 24644
00:24:04.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.119639 25140 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:04.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.119969 25140 heartbeater.cc:461] Registering TS with master...
00:24:04.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.120990 25140 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.124912 24411 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.126819 25141 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.126961 25141 heartbeater.cc:461] Registering TS with master...
00:24:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.127466 25141 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.128852 24268 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.130404 24268 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.129:39473
00:24:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.131824 25142 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.131946 25142 heartbeater.cc:461] Registering TS with master...
00:24:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.132396 25142 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:04.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.134013 24341 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:04.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.135076 25141 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.152920 24766 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:04.254377 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 24644
00:24:04.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:04.271981 24793 connection.cc:573] server connection from 127.23.168.130:53787 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:04.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:04.273 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] handling channelInactive
00:24:04.273 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state READY due to: connection closed
00:24:04.294 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:04.296 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.296 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.296 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.317 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:04.318 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:04.318 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=3, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [23ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@273493864(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@332c5611 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72cf7f41 -> wakeup thread Time-limited test)) after lookup
00:24:04.319 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:04.320 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.320 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.320 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:04.338 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:04.338 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=5, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [23ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [45ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [47ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [47ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@273493864(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@332c5611 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72cf7f41 -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206))) after lookup
00:24:04.339 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:04.339 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.339 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.340 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.397 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:04.397 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:04.398 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=7, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=125), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [23ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [45ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [47ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [47ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [65ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [67ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [67ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [121ms] refreshing cache from master, [121ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [124ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@273493864(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@332c5611 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72cf7f41 -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578))) after lookup
00:24:04.399 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:04.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.400 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.617 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:04.618 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:04.618 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=9, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=345), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [23ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [45ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [47ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [47ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [65ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [67ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [67ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [121ms] refreshing cache from master, [121ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [124ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [125ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [127ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [127ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [341ms] refreshing cache from master, [341ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [344ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@273493864(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@332c5611 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578) -> (continuation of Deferred@883687597 after retry RPC@1692449900), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72cf7f41 -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578) -> (continuation of Deferred@883687597 after retry RPC@1692449900))) after lookup
00:24:04.619 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:04.621 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.621 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.621 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.659 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:04.659 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:04.660 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=11, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=386), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [23ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [45ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [47ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [47ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [65ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [67ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [67ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [121ms] refreshing cache from master, [121ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [124ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [125ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [127ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [127ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [341ms] refreshing cache from master, [341ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [344ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [346ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [348ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [348ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [382ms] refreshing cache from master, [382ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [385ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@273493864(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@332c5611 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578) -> (continuation of Deferred@883687597 after retry RPC@1692449900) -> (continuation of Deferred@1533844781 after retry RPC@669905563), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72cf7f41 -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578) -> (continuation of Deferred@883687597 after retry RPC@1692449900) -> (continuation of Deferred@1533844781 after retry RPC@669905563))) after lookup
00:24:04.660 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:04.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:04.662 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.416747 25155 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.417003 25155 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.418684 25155 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:05.421479 25029 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:05.423578 25029 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:05.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.427285 24860 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.428023 25027 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.428467 25155 raft_consensus.cc:2749] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
00:24:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.755766 25160 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9e8a7a02fccc4f5bb47477a8c99ae1e8)
00:24:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.756011 25160 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.756814 25160 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:05.757798 24796 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111) [suppressed 1 similar messages]
00:24:05.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:05.759477 24796 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.766063 25095 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381" is_pre_election: true
00:24:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.766448 25095 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 1.
00:24:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.767338 24795 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.767681 25160 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Leader pre-election won for term 4
00:24:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.767825 25160 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 9e8a7a02fccc4f5bb47477a8c99ae1e8)
00:24:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.767932 25160 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 3 FOLLOWER]: Advancing to term 4
00:24:05.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.769963 25160 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:05.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.770670 25160 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 4 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.771248 25095 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.771448 25095 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 1 FOLLOWER]: Advancing to term 4
00:24:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:05.772226 24796 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.775367 25095 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 4.
00:24:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.775789 24795 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.776062 25160 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Leader election won for term 4
00:24:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.776603 25160 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 LEADER]: Becoming Leader. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Running, Role: LEADER
00:24:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.777114 25160 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:05.781139 24268 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 reported cstate change: term changed from 3 to 4, leader changed from 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130) to 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131). New cstate: current_term: 4 leader_uuid: "81d9cd709bc146289a569ff26e47b3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: HEALTHY } } }
00:24:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:06.172940 24796 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 -> Peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Couldn't send request to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:06.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:06.188721 25095 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Refusing update from remote peer 81d9cd709bc146289a569ff26e47b3c8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
00:24:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:06.189719 25160 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
00:24:07.558 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:07.558 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:07.559 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=13, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3286), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [23ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [45ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [47ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [47ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [65ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [67ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [67ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [121ms] refreshing cache from master, [121ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [124ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [125ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [127ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [127ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [341ms] refreshing cache from master, [341ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [344ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [346ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [348ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [348ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [382ms] refreshing cache from master, [382ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [385ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [387ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [389ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [389ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [3281ms] refreshing cache from master, [3282ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [3285ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@273493864(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@332c5611 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578) -> (continuation of Deferred@883687597 after retry RPC@1692449900) -> (continuation of Deferred@1533844781 after retry RPC@669905563) -> (continuation of Deferred@1865390636 after retry RPC@664720504), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72cf7f41 -> wakeup thread Time-limited test -> (continuation of Deferred@1828579967 after retry RPC@1698805206) -> (continuation of Deferred@1705787524 after retry RPC@220964578) -> (continuation of Deferred@883687597 after retry RPC@1692449900) -> (continuation of Deferred@1533844781 after retry RPC@669905563) -> (continuation of Deferred@1865390636 after retry RPC@664720504))) after lookup
00:24:07.560 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:07.561 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] Successfully connected to peer
00:24:07.578 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1cb357e9, L:/127.0.0.1:52186 - R:/127.23.168.131:37089] using TOKEN/null
00:24:07.613 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.130:34095
00:24:07.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.615347 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:34095
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36067
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.952119 25197 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.952464 25197 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.952587 25197 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.960930 25197 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.961158 25197 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.130
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.970983 25197 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:34095
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36067
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.973111 25197 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.975026 25197 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.985013 25204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.985098 25207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:07.985271 25205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.986116 25197 server_base.cc:1061] running on GCE node
00:24:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.986727 25197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:07.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.987649 25197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.988879 25197 hybrid_clock.cc:648] HybridClock initialized: now 1783470247988843 us; error 31 us; skew 500 ppm
00:24:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.989260 25197 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:07.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.991357 25197 webserver.cc:534] Webserver started at http://127.23.168.130:36067/ using document root <none> and password file <none>
00:24:07.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.992107 25197 fs_manager.cc:362] Metadata directory not provided
00:24:07.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.992250 25197 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:07.997937 25197 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.000s	sys 0.002s
00:24:08.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.000689 25213 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.002018 25197 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
00:24:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.002203 25197 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.002947 25197 fs_report.cc:389] FS layout report
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/data
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.021322 25197 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.022166 25197 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.022397 25197 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.023316 25197 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.025303 25220 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.031145 25197 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.031225 25197 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.031318 25197 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.034333 25197 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.034416 25197 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
00:24:08.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.034909 25220 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap starting.
00:24:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.058302 25220 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log is configured to *not* fsync() on all Append() calls
00:24:08.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.075390 25220 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:08.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.076038 25220 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap complete.
00:24:08.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.077167 25220 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent bootstrapping tablet: real 0.043s	user 0.024s	sys 0.015s
00:24:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.085273 25220 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.086066 25220 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Initialized, Role: FOLLOWER
00:24:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.087195 25220 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.089517 25220 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent starting tablet: real 0.012s	user 0.011s	sys 0.002s
00:24:08.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.090178 25197 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.130:34095
00:24:08.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.090246 25328 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.130:34095 every 8 connection(s)
00:24:08.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.091854 25197 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:08.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.092869 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 25197
00:24:08.120 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:08.123 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.131:37089
00:24:08.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:24:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.123880 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 24778
00:24:08.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.151450 24900 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:08.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.156349 25329 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.156798 25329 heartbeater.cc:461] Registering TS with master...
00:24:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.158313 25329 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:08.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.160120 25330 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:08.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.160267 25330 heartbeater.cc:461] Registering TS with master...
00:24:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.160890 25330 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:08.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.161741 24411 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:08.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.162222 24267 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.163901 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.130:42855
00:24:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.164011 25333 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.164143 25333 heartbeater.cc:461] Registering TS with master...
00:24:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.164678 25333 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.165895 24341 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:08.258574 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 24778
00:24:08.278 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] handling channelInactive
00:24:08.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state READY due to: connection closed
00:24:08.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:08.278573 25344 negotiation.cc:336] Failed RPC negotiation. Trace:
00:24:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:08.257852 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.23.168.131:56479 (local address 127.23.168.130:34095)
00:24:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:08.262430 (+  4578us) server_negotiation.cc:207] Beginning negotiation
00:24:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:08.262438 (+     8us) server_negotiation.cc:400] Waiting for connection header
00:24:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:08.277031 (+ 14593us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.23.168.131:56479: BlockingRecv error: recv got EOF from 127.23.168.131:56479 (error 108)
00:24:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4445,"thread_start_us":4332,"threads_started":1}
00:24:08.279 [DEBUG - Time-limited test] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:08.280 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.280 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.280 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.298 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:08.298 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:08.299 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [2ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [16ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [20ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1027669904(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@61788b97 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46c673e0 -> wakeup thread Time-limited test)) after lookup
00:24:08.299 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:08.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.300 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.301 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.317 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:08.318 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:08.318 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [2ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [16ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [20ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [21ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [23ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [23ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [39ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1027669904(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@61788b97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46c673e0 -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574))) after lookup
00:24:08.319 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:08.319 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.320 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.320 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:08.338 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:08.338 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [2ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [16ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [20ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [21ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [23ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [23ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [39ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [40ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [42ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [42ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [59ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1027669904(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@61788b97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46c673e0 -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823))) after lookup
00:24:08.338 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:08.339 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.340 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.340 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.397 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:08.397 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:08.397 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=119), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [2ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [16ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [20ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [21ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [23ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [23ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [39ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [40ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [42ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [42ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [59ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [60ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [62ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [62ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [119ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1027669904(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@61788b97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823) -> (continuation of Deferred@1100117482 after retry RPC@1569499047), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46c673e0 -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823) -> (continuation of Deferred@1100117482 after retry RPC@1569499047))) after lookup
00:24:08.398 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:08.399 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.399 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.399 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.617 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:08.618 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:08.618 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=339), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [2ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [16ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [20ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [21ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [23ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [23ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [39ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [40ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [42ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [42ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [59ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [60ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [62ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [62ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [119ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [120ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [121ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [121ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [336ms] refreshing cache from master, [336ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [339ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1027669904(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@61788b97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823) -> (continuation of Deferred@1100117482 after retry RPC@1569499047) -> (continuation of Deferred@357653216 after retry RPC@769156722), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46c673e0 -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823) -> (continuation of Deferred@1100117482 after retry RPC@1569499047) -> (continuation of Deferred@357653216 after retry RPC@769156722))) after lookup
00:24:08.619 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:08.620 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.620 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:08.621 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.166826 25330 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.546070 25345 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 81d9cd709bc146289a569ff26e47b3c8)
00:24:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.546389 25345 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.547195 25345 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:09.549939 25027 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.555567 25283 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" is_pre_election: true
00:24:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.556041 25283 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 3.
00:24:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.557011 25029 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8
00:24:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.557302 25345 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Leader pre-election won for term 5
00:24:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.557432 25345 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 81d9cd709bc146289a569ff26e47b3c8)
00:24:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.557503 25345 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 4 FOLLOWER]: Advancing to term 5
00:24:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.559662 25345 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.560290 25345 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 5 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.560711 25283 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.560936 25283 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 3 FOLLOWER]: Advancing to term 5
00:24:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:09.562099 25027 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.564868 25283 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 5.
00:24:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.565269 25029 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8
00:24:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.565553 25345 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 FOLLOWER]: Leader election won for term 5
00:24:09.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.565964 25345 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 LEADER]: Becoming Leader. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Running, Role: LEADER
00:24:09.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.566780 25345 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.570454 24267 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 reported cstate change: term changed from 4 to 5, leader changed from 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131) to af84f760f77249aa81d9789e1b93e381 (127.23.168.129). New cstate: current_term: 5 leader_uuid: "af84f760f77249aa81d9789e1b93e381" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.994500 25283 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Refusing update from remote peer af84f760f77249aa81d9789e1b93e381: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
00:24:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:09.995281 25345 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
00:24:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.010529 25027 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 -> Peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Couldn't send request to peer 81d9cd709bc146289a569ff26e47b3c8. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:10.238 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:10.238 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:10.239 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1960), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [2ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [16ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [20ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [21ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [23ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [23ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [39ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [40ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [42ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [42ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [59ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [60ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [62ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [62ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [119ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [120ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [121ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [121ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [336ms] refreshing cache from master, [336ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [339ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [341ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [342ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [343ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1956ms] refreshing cache from master, [1956ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1960ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1027669904(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@61788b97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823) -> (continuation of Deferred@1100117482 after retry RPC@1569499047) -> (continuation of Deferred@357653216 after retry RPC@769156722) -> (continuation of Deferred@393625671 after retry RPC@2045988653), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46c673e0 -> wakeup thread Time-limited test -> (continuation of Deferred@1313682396 after retry RPC@1004605574) -> (continuation of Deferred@282179355 after retry RPC@587718823) -> (continuation of Deferred@1100117482 after retry RPC@1569499047) -> (continuation of Deferred@357653216 after retry RPC@769156722) -> (continuation of Deferred@393625671 after retry RPC@2045988653))) after lookup
00:24:10.239 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:10.240 [DEBUG - kudu-nio-0] (Connection.java:757) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] Successfully connected to peer
00:24:10.262 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5f5a71e5, L:/127.0.0.1:38988 - R:/127.23.168.129:40013] using TOKEN/null
00:24:10.285 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.131:37089
00:24:10.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:24:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.286568 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:24:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:37089
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32819
00:24:10.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:24:10.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:10.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:10.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:10.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.601715 25027 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111) [suppressed 3 similar messages]
00:24:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.627424 25373 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.627734 25373 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.627828 25373 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:10.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.636332 25373 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:10.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.636498 25373 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.131
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.646724 25373 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:37089
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32819
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:10.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.648311 25373 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:10.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.650100 25373 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.659623 25381 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.660386 25380 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.660646 25383 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:10.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.661001 25373 server_base.cc:1061] running on GCE node
00:24:10.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.661665 25373 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.662791 25373 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:10.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.664026 25373 hybrid_clock.cc:648] HybridClock initialized: now 1783470250663974 us; error 42 us; skew 500 ppm
00:24:10.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.664336 25373 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.666538 25373 webserver.cc:534] Webserver started at http://127.23.168.131:32819/ using document root <none> and password file <none>
00:24:10.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.667207 25373 fs_manager.cc:362] Metadata directory not provided
00:24:10.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.667349 25373 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.672621 25373 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.002s	sys 0.003s
00:24:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.675405 25389 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:10.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.676698 25373 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.003s
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.676823 25373 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:57 on dist-test-slave-4ccz"
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.677508 25373 fs_report.cc:389] FS layout report
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/data
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.693851 25373 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.694693 25373 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.694926 25373 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.695784 25373 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.697672 25396 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.703446 25373 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.703567 25373 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.703727 25373 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.707603 25373 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.707690 25373 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
00:24:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.708261 25396 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap starting.
00:24:10.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.729879 25396 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Log is configured to *not* fsync() on all Append() calls
00:24:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.752020 25396 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.752661 25396 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap complete.
00:24:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.753834 25396 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent bootstrapping tablet: real 0.046s	user 0.028s	sys 0.016s
00:24:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.757407 25373 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.131:37089
00:24:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.757503 25503 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.131:37089 every 8 connection(s)
00:24:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.759021 25373 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.760520 25396 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.761586 25396 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Initialized, Role: FOLLOWER
00:24:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.763928 25396 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.765226 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 25373
00:24:10.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.769193 25396 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent starting tablet: real 0.015s	user 0.011s	sys 0.003s
00:24:10.792 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:10.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.801138 25505 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:10.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.801476 25505 heartbeater.cc:461] Registering TS with master...
00:24:10.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.802666 25505 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:10.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.806171 24267 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:10.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.806346 25506 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:10.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.806471 25506 heartbeater.cc:461] Registering TS with master...
00:24:10.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.807016 25506 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:10.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.807761 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.131:37623
00:24:10.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.808451 24340 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:10.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.810956 25505 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:10.812 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.129:40013
00:24:10.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:10.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.813603 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 25009
00:24:10.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.822468 25504 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:10.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.822633 25504 heartbeater.cc:461] Registering TS with master...
00:24:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.822948 25504 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.824571 24411 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.825733 25135 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:10.926906 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 25009
00:24:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:10.944630 25215 connection.cc:573] server connection from 127.23.168.129:40333 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:10.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:10.945 [DEBUG - kudu-nio-0] (Connection.java:239) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] handling channelInactive
00:24:10.946 [DEBUG - kudu-nio-0] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state READY due to: connection closed
00:24:10.946 [DEBUG - Time-limited test] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:10.947 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.947 [DEBUG - kudu-nio-1] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.947 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.957 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:10.957 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:10.957 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test)) after lookup
00:24:10.958 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:10.959 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.959 [DEBUG - kudu-nio-2] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.959 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.977 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:10.977 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:10.977 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [12ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [13ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [13ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [30ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733))) after lookup
00:24:10.978 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:10.979 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.979 [DEBUG - kudu-nio-3] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:10.979 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.017 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:11.017 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:11.018 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [12ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [13ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [13ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [30ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [31ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [33ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [33ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936))) after lookup
00:24:11.018 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:11.019 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.020 [DEBUG - kudu-nio-4] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.020 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.057 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:11.058 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:11.058 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=112), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [12ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [13ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [13ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [30ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [31ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [33ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [33ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [111ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781))) after lookup
00:24:11.059 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:11.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.060 [DEBUG - kudu-nio-5] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.060 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.298 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:11.298 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:11.299 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=353), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [12ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [13ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [13ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [30ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [31ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [33ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [33ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [111ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [113ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [114ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [114ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [352ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781) -> (continuation of Deferred@942128027 after retry RPC@54368614), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781) -> (continuation of Deferred@942128027 after retry RPC@54368614))) after lookup
00:24:11.300 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:11.302 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.302 [DEBUG - kudu-nio-6] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:11.302 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:12.397 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:12.397 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:12.398 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1452), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [12ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [13ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [13ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [30ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [31ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [33ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [33ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [111ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [113ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [114ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [114ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [352ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [354ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [356ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [356ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1448ms] refreshing cache from master, [1448ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1451ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781) -> (continuation of Deferred@942128027 after retry RPC@54368614) -> (continuation of Deferred@311553784 after retry RPC@2106110821), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781) -> (continuation of Deferred@942128027 after retry RPC@54368614) -> (continuation of Deferred@311553784 after retry RPC@2106110821))) after lookup
00:24:12.399 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:12.400 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:12.400 [DEBUG - kudu-nio-7] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:12.400 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.426028 25519 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.426327 25519 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.428049 25519 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.428119 25520 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.428334 25520 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:12.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.429913 25392 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:12.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.431562 25392 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.432102 25520 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.433084 25216 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:12.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.434800 25216 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:12.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.436790 25283 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" is_pre_election: true
00:24:12.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.437079 25283 raft_consensus.cc:2393] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 81d9cd709bc146289a569ff26e47b3c8 in current term 5: Already voted for candidate af84f760f77249aa81d9789e1b93e381 in this term.
00:24:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.437956 25393 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381
00:24:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.438359 25519 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 4 FOLLOWER]: Advancing to term 5
00:24:12.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.441946 25459 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.443465 25519 raft_consensus.cc:2749] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 5 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
00:24:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.443804 25459 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 5.
00:24:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.445276 25215 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.445690 25520 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Leader pre-election won for term 6
00:24:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.445817 25520 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Starting leader election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:12.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.445888 25520 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 5 FOLLOWER]: Advancing to term 6
00:24:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.447924 25520 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.448522 25520 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 6 election: Requested vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:12.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.449079 25459 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:12.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.449260 25459 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 5 FOLLOWER]: Advancing to term 6
00:24:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.449855 25216 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.451447 25459 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 6.
00:24:12.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.451943 25215 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:12.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.452260 25520 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 FOLLOWER]: Leader election won for term 6
00:24:12.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.452644 25520 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 LEADER]: Becoming Leader. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Running, Role: LEADER
00:24:12.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.453168 25520 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.456631 24267 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 reported cstate change: term changed from 5 to 6, leader changed from af84f760f77249aa81d9789e1b93e381 (127.23.168.129) to 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130). New cstate: current_term: 6 leader_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:12.598 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:12.598 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:12.599 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1652), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [11ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [12ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [13ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [13ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [30ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [31ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [33ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [33ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [72ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [111ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [113ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [114ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [114ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [352ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [354ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [356ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [356ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1448ms] refreshing cache from master, [1448ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1451ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [1452ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1454ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1454ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1649ms] refreshing cache from master, [1649ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1652ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@42890499(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@6167c858 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781) -> (continuation of Deferred@942128027 after retry RPC@54368614) -> (continuation of Deferred@311553784 after retry RPC@2106110821) -> (continuation of Deferred@459648845 after retry RPC@600712415), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6992bb39 -> wakeup thread Time-limited test -> (continuation of Deferred@675408962 after retry RPC@739734733) -> (continuation of Deferred@1312350750 after retry RPC@1503548936) -> (continuation of Deferred@658755595 after retry RPC@205974781) -> (continuation of Deferred@942128027 after retry RPC@54368614) -> (continuation of Deferred@311553784 after retry RPC@2106110821) -> (continuation of Deferred@459648845 after retry RPC@600712415))) after lookup
00:24:12.599 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:12.600 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] Successfully connected to peer
00:24:12.619 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x978f4396, L:/127.0.0.1:33670 - R:/127.23.168.130:34095] using TOKEN/null
00:24:12.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.625499 25459 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Refusing update from remote peer 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 18. (index mismatch)
00:24:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.625907 25216 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 -> Peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Couldn't send request to peer af84f760f77249aa81d9789e1b93e381. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.626477 25529 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
00:24:12.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.635881 25542 mvcc.cc:204] Tried to move back new op lower bound from 7305094154745360384 to 7305094154051735552. Current Snapshot: MvccSnapshot[applied={T|T < 7305094154745360384}]
00:24:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.636916 25543 mvcc.cc:204] Tried to move back new op lower bound from 7305094154745360384 to 7305094154051735552. Current Snapshot: MvccSnapshot[applied={T|T < 7305094154745360384}]
00:24:12.657 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.129:40013
00:24:12.658 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:12.658838 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.985181 25550 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.985492 25550 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.985575 25550 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:12.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.993598 25550 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:12.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:12.993762 25550 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.129
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.003839 25550 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.005924 25550 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.007702 25550 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:13.017134 25556 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:13.017521 25555 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:13.018193 25558 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.018589 25550 server_base.cc:1061] running on GCE node
00:24:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.019191 25550 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.020219 25550 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.021459 25550 hybrid_clock.cc:648] HybridClock initialized: now 1783470253021407 us; error 44 us; skew 500 ppm
00:24:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.021809 25550 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:13.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.024063 25550 webserver.cc:534] Webserver started at http://127.23.168.129:32839/ using document root <none> and password file <none>
00:24:13.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.024834 25550 fs_manager.cc:362] Metadata directory not provided
00:24:13.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.024964 25550 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:13.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.030376 25550 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.000s
00:24:13.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.033202 25564 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:13.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.034580 25550 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.034768 25550 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.035574 25550 fs_report.cc:389] FS layout report
00:24:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/data
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:13.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.049346 25550 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.050302 25550 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.050632 25550 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.051612 25550 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.053710 25571 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.060938 25550 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.061062 25550 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
00:24:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.061200 25550 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:13.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.064337 25550 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:13.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.064478 25550 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.001s	sys 0.000s
00:24:13.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.064776 25571 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap starting.
00:24:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.087320 25571 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Log is configured to *not* fsync() on all Append() calls
00:24:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.116357 25571 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.117295 25550 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.129:40013
00:24:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.117327 25678 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.129:40013 every 8 connection(s)
00:24:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.117520 25571 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap complete.
00:24:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.119046 25571 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent bootstrapping tablet: real 0.055s	user 0.036s	sys 0.016s
00:24:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.119330 25550 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.125922 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 25550
00:24:13.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:13.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.128773 25571 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:13.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.129705 25571 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Initialized, Role: FOLLOWER
00:24:13.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.130543 25571 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.143784 25571 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent starting tablet: real 0.025s	user 0.012s	sys 0.007s
00:24:13.147 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:13.148 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.130:34095
00:24:13.148 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:24:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.149194 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 25197
00:24:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.204177 25679 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.204631 25679 heartbeater.cc:461] Registering TS with master...
00:24:13.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.206092 25679 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.210333 24411 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.220304 25680 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.220489 25680 heartbeater.cc:461] Registering TS with master...
00:24:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.221146 25680 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:13.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.222782 25681 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:13.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.222945 25681 heartbeater.cc:461] Registering TS with master...
00:24:13.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.223060 24267 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:13.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.223455 25681 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:13.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.224614 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.129:41769
00:24:13.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.225059 24340 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.227977 25680 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:13.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.231429 25323 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:13.328788 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 25197
00:24:13.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:13.343 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] handling channelInactive
00:24:13.343 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state READY due to: connection closed
00:24:13.344 [DEBUG - Time-limited test] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.345 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:13.358 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:13.358 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test)) after lookup
00:24:13.359 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.360 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.360 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.360 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.377 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:13.377 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:13.377 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [16ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [17ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [17ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207))) after lookup
00:24:13.378 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.379 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.379 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.379 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:13.418 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:13.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [16ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [17ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [17ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [36ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [36ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882))) after lookup
00:24:13.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.420 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.420 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.420 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.437 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:13.438 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:13.438 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=94), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [16ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [17ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [17ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [36ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [36ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [77ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [77ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [94ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319))) after lookup
00:24:13.439 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.440 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.440 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.440 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.697 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:13.697 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:13.698 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=354), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [16ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [17ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [17ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [36ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [36ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [77ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [77ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [94ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [95ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [97ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [97ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [354ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319) -> (continuation of Deferred@1830047113 after retry RPC@720558436), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319) -> (continuation of Deferred@1830047113 after retry RPC@720558436))) after lookup
00:24:13.698 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.699 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.700 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.700 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.937 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:13.938 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:13.938 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=594), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [16ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [17ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [17ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [36ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [36ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [77ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [77ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [94ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [95ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [97ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [97ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [354ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [355ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [357ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [357ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [591ms] refreshing cache from master, [591ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [594ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319) -> (continuation of Deferred@1830047113 after retry RPC@720558436) -> (continuation of Deferred@2074395945 after retry RPC@1201740480), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319) -> (continuation of Deferred@1830047113 after retry RPC@720558436) -> (continuation of Deferred@2074395945 after retry RPC@1201740480))) after lookup
00:24:13.939 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:13.940 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.940 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:13.940 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.560376 25694 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 9e8a7a02fccc4f5bb47477a8c99ae1e8)
00:24:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.560628 25694 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.561404 25694 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:14.563550 25393 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.569639 25634 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381" is_pre_election: true
00:24:14.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.570159 25634 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 5.
00:24:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.571090 25392 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.571412 25694 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Leader pre-election won for term 7
00:24:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.571529 25694 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 9e8a7a02fccc4f5bb47477a8c99ae1e8)
00:24:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.571589 25694 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 6 FOLLOWER]: Advancing to term 7
00:24:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.573518 25694 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.574190 25694 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 7 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.574780 25634 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.575002 25634 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 5 FOLLOWER]: Advancing to term 7
00:24:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:14.575804 25393 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.578871 25634 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 7.
00:24:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.579270 25392 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.579583 25694 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 FOLLOWER]: Leader election won for term 7
00:24:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.580053 25694 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 LEADER]: Becoming Leader. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Running, Role: LEADER
00:24:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.580569 25694 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:14.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:14.584219 24267 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 reported cstate change: term changed from 6 to 7, leader changed from 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130) to 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131). New cstate: current_term: 7 leader_uuid: "81d9cd709bc146289a569ff26e47b3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: HEALTHY } } }
00:24:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.105386 25634 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Refusing update from remote peer 81d9cd709bc146289a569ff26e47b3c8: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
00:24:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.106240 25712 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
00:24:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.139976 25393 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 -> Peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Couldn't send request to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:15.158 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:15.158 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:15.159 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1815), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [2ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [16ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [17ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [17ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [36ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [36ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [77ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [77ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [94ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [95ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [97ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [97ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [354ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [355ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [357ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [357ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [591ms] refreshing cache from master, [591ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [594ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [596ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [597ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [597ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1811ms] refreshing cache from master, [1812ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1814ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1598355145(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@d7ed922 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319) -> (continuation of Deferred@1830047113 after retry RPC@720558436) -> (continuation of Deferred@2074395945 after retry RPC@1201740480) -> (continuation of Deferred@603864304 after retry RPC@1816548693), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70277f18 -> wakeup thread Time-limited test -> (continuation of Deferred@1183693828 after retry RPC@313572207) -> (continuation of Deferred@1375496353 after retry RPC@1174456882) -> (continuation of Deferred@934743401 after retry RPC@1998979319) -> (continuation of Deferred@1830047113 after retry RPC@720558436) -> (continuation of Deferred@2074395945 after retry RPC@1201740480) -> (continuation of Deferred@603864304 after retry RPC@1816548693))) after lookup
00:24:15.159 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:15.160 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] Successfully connected to peer
00:24:15.180 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcbf950ae, L:/127.0.0.1:41472 - R:/127.23.168.131:37089] using TOKEN/null
00:24:15.206 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.130:34095
00:24:15.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.208405 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:34095
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36067
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:15.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.555897 25727 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:15.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.556344 25727 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:15.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.556468 25727 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.565032 25727 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.565196 25727 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.130
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.575119 25727 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:34095
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36067
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:24:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.576795 25727 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:15.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.578683 25727 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.588824 25733 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.589017 25732 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.589776 25735 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.590569 25727 server_base.cc:1061] running on GCE node
00:24:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.591104 25727 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:15.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.592167 25727 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.593380 25727 hybrid_clock.cc:648] HybridClock initialized: now 1783470255593323 us; error 53 us; skew 500 ppm
00:24:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.593691 25727 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.595974 25727 webserver.cc:534] Webserver started at http://127.23.168.130:36067/ using document root <none> and password file <none>
00:24:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.596642 25727 fs_manager.cc:362] Metadata directory not provided
00:24:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.596729 25727 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.602262 25727 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
00:24:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.605134 25741 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.606462 25727 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.606642 25727 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.607455 25727 fs_report.cc:389] FS layout report
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/data
00:24:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.631034 25727 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:15.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.632005 25727 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:15.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.632359 25727 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.633385 25727 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.635627 25749 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.641621 25727 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.641737 25727 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.641889 25727 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:15.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.644941 25727 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:15.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.645072 25727 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
00:24:15.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.645478 25749 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap starting.
00:24:15.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.671519 25749 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log is configured to *not* fsync() on all Append() calls
00:24:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.689158 25727 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.130:34095
00:24:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.689190 25856 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.130:34095 every 8 connection(s)
00:24:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.690979 25727 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:15.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.698774 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 25727
00:24:15.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.705926 25749 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.706880 25749 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap complete.
00:24:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.708509 25749 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent bootstrapping tablet: real 0.063s	user 0.047s	sys 0.012s
00:24:15.713 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:15.713 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.131:37089
00:24:15.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:24:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.714561 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 25373
00:24:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.715744 25749 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.717082 25749 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Initialized, Role: FOLLOWER
00:24:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.717947 25749 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.726567 25749 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent starting tablet: real 0.018s	user 0.016s	sys 0.004s
00:24:15.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.748911 25499 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.766036 25859 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.766462 25859 heartbeater.cc:461] Registering TS with master...
00:24:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.767160 25857 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.767308 25857 heartbeater.cc:461] Registering TS with master...
00:24:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.767498 25859 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.767702 25857 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.766052 25858 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.769181 25858 heartbeater.cc:461] Registering TS with master...
00:24:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.769584 25858 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.769941 24411 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.770737 24267 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.770798 24340 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.772145 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.130:53373
00:24:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.774892 25858 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:15.863869 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 25373
00:24:15.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:15.881844 25566 connection.cc:573] server connection from 127.23.168.131:35189 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:15.882 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] handling channelInactive
00:24:15.882 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state READY due to: connection closed
00:24:15.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:15.883 [DEBUG - Time-limited test] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:15.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.884 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:15.897 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:15.898 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1279673261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@12744b8d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a38261 -> wakeup thread Time-limited test)) after lookup
00:24:15.898 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:15.899 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.899 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.900 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.917 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:15.917 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:15.917 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [17ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [17ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1279673261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@12744b8d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a38261 -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725))) after lookup
00:24:15.918 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:15.919 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.919 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.919 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.958 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:15.958 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:15.958 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [17ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [17ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [36ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1279673261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@12744b8d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a38261 -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550))) after lookup
00:24:15.959 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:15.960 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.960 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:15.960 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:16.077 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:16.077 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:16.077 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=194), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [17ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [17ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [36ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [77ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [77ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [193ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1279673261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@12744b8d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550) -> (continuation of Deferred@1476341425 after retry RPC@1271891582), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a38261 -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550) -> (continuation of Deferred@1476341425 after retry RPC@1271891582))) after lookup
00:24:16.078 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:16.079 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:16.079 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:16.079 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:16.458 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:16.458 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:16.459 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=575), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [17ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [17ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [36ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [77ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [77ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [193ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [195ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [196ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [196ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [572ms] refreshing cache from master, [572ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [575ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1279673261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@12744b8d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550) -> (continuation of Deferred@1476341425 after retry RPC@1271891582) -> (continuation of Deferred@48314335 after retry RPC@891989387), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a38261 -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550) -> (continuation of Deferred@1476341425 after retry RPC@1271891582) -> (continuation of Deferred@48314335 after retry RPC@891989387))) after lookup
00:24:16.459 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:16.460 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:16.461 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:16.461 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:17.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.366209 25872 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 81d9cd709bc146289a569ff26e47b3c8)
00:24:17.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.366559 25872 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.368124 25872 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:17.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:17.369968 25566 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:17.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:17.375159 25566 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.377226 25812 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" is_pre_election: true
00:24:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.377653 25812 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 6.
00:24:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.378589 25568 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8
00:24:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.378966 25872 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Leader pre-election won for term 8
00:24:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.379103 25872 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 81d9cd709bc146289a569ff26e47b3c8)
00:24:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.379207 25872 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 7 FOLLOWER]: Advancing to term 8
00:24:17.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.381424 25872 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.382102 25872 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 8 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.382623 25812 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.382836 25812 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 6 FOLLOWER]: Advancing to term 8
00:24:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:17.383522 25566 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.386767 25812 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 8.
00:24:17.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.387197 25568 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8
00:24:17.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.387507 25872 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Leader election won for term 8
00:24:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.387871 25872 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 LEADER]: Becoming Leader. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Running, Role: LEADER
00:24:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.388373 25872 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.392035 24267 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 reported cstate change: term changed from 7 to 8, leader changed from 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131) to af84f760f77249aa81d9789e1b93e381 (127.23.168.129). New cstate: current_term: 8 leader_uuid: "af84f760f77249aa81d9789e1b93e381" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:17.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:17.677 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:17.678 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1794), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [17ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [17ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [36ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [36ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [74ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [77ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [77ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [193ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [195ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [196ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [196ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [572ms] refreshing cache from master, [572ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [575ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [576ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [578ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [578ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1791ms] refreshing cache from master, [1791ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1794ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1279673261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@12744b8d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550) -> (continuation of Deferred@1476341425 after retry RPC@1271891582) -> (continuation of Deferred@48314335 after retry RPC@891989387) -> (continuation of Deferred@1712408433 after retry RPC@112720077), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a38261 -> wakeup thread Time-limited test -> (continuation of Deferred@1615540929 after retry RPC@1986161725) -> (continuation of Deferred@843054912 after retry RPC@1473871550) -> (continuation of Deferred@1476341425 after retry RPC@1271891582) -> (continuation of Deferred@48314335 after retry RPC@891989387) -> (continuation of Deferred@1712408433 after retry RPC@112720077))) after lookup
00:24:17.678 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:17.679 [DEBUG - kudu-nio-7] (Connection.java:757) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] Successfully connected to peer
00:24:17.701 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x8cfa4bd3, L:/127.0.0.1:39662 - R:/127.23.168.129:40013] using TOKEN/null
00:24:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.707054 25812 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Refusing update from remote peer af84f760f77249aa81d9789e1b93e381: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 26. (index mismatch)
00:24:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:17.707425 25566 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 -> Peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Couldn't send request to peer 81d9cd709bc146289a569ff26e47b3c8. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.707901 25872 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
00:24:17.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.718694 25891 mvcc.cc:204] Tried to move back new op lower bound from 7305094175558467584 to 7305094174266429440. Current Snapshot: MvccSnapshot[applied={T|T < 7305094175558467584}]
00:24:17.738 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.131:37089
00:24:17.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:17.739240 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:37089
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32819
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:18.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.069648 25900 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:18.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.070022 25900 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:18.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.070151 25900 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.078506 25900 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.078732 25900 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.131
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.088797 25900 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:37089
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32819
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:24:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.090873 25900 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.092741 25900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.102094 25905 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.102133 25906 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:18.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.103066 25908 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:18.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.103434 25900 server_base.cc:1061] running on GCE node
00:24:18.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.103968 25900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:18.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.104825 25900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:18.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.106060 25900 hybrid_clock.cc:648] HybridClock initialized: now 1783470258105995 us; error 50 us; skew 500 ppm
00:24:18.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.106334 25900 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.108371 25900 webserver.cc:534] Webserver started at http://127.23.168.131:32819/ using document root <none> and password file <none>
00:24:18.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.109157 25900 fs_manager.cc:362] Metadata directory not provided
00:24:18.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.109256 25900 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:18.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.114706 25900 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
00:24:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.117489 25914 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:18.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.119021 25900 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.119175 25900 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:57 on dist-test-slave-4ccz"
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.120067 25900 fs_report.cc:389] FS layout report
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/data
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.135823 25900 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.136540 25900 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.136785 25900 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:18.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.137616 25900 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.139544 25921 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:18.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.145494 25900 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:18.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.145576 25900 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:18.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.145668 25900 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.148584 25900 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.148699 25900 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
00:24:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.149252 25921 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap starting.
00:24:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.180373 25921 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Log is configured to *not* fsync() on all Append() calls
00:24:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.187191 25900 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.131:37089
00:24:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.187222 26028 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.131:37089 every 8 connection(s)
00:24:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.188901 25900 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:18.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.195935 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 25900
00:24:18.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.225749 25921 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.226668 25921 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap complete.
00:24:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.228695 25921 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent bootstrapping tablet: real 0.080s	user 0.043s	sys 0.029s
00:24:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.239112 25921 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.240456 25921 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Initialized, Role: FOLLOWER
00:24:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.241374 25921 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:18.246 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:18.247 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.129:40013
00:24:18.247 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.247941 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 25550
00:24:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.251420 25921 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent starting tablet: real 0.023s	user 0.013s	sys 0.009s
00:24:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.258426 26029 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.258787 26029 heartbeater.cc:461] Registering TS with master...
00:24:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.259817 26029 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:18.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.261020 25674 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.262365 26031 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.262529 26031 heartbeater.cc:461] Registering TS with master...
00:24:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.263088 26031 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.264003 26030 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.264127 26030 heartbeater.cc:461] Registering TS with master...
00:24:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.264366 24411 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.264552 26030 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.264599 24341 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:18.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.266033 24267 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.267904 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.131:55237
00:24:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.270673 26030 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:18.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:18.362107 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 25550
00:24:18.381 [DEBUG - kudu-nio-7] (Connection.java:239) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] handling channelInactive
00:24:18.381 [DEBUG - kudu-nio-7] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state READY due to: connection closed
00:24:18.382 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:18.381898 26046 negotiation.cc:336] Failed RPC negotiation. Trace:
00:24:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:18.362262 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.23.168.129:39363 (local address 127.23.168.131:37089)
00:24:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:18.368050 (+  5788us) server_negotiation.cc:207] Beginning negotiation
00:24:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:18.368057 (+     7us) server_negotiation.cc:400] Waiting for connection header
00:24:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:18.380535 (+ 12478us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.23.168.129:39363: BlockingRecv error: recv got EOF from 127.23.168.129:39363 (error 108)
00:24:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5616,"thread_start_us":5460,"threads_started":1}
00:24:18.382 [DEBUG - Time-limited test] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:18.383 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.383 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.397 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:18.397 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:18.397 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1937552651(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@4a344a39 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@409c8882 -> wakeup thread Time-limited test)) after lookup
00:24:18.398 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:18.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.399 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.417 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:18.418 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:18.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [17ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [17ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [35ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1937552651(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@4a344a39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@409c8882 -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728))) after lookup
00:24:18.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:18.419 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.420 [DEBUG - kudu-nio-2] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.420 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.457 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:18.457 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:18.458 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [17ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [17ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [35ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [36ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [38ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [38ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [75ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1937552651(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@4a344a39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@409c8882 -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811))) after lookup
00:24:18.458 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:18.459 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.459 [DEBUG - kudu-nio-3] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.460 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.478 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:18.478 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:18.478 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=96), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [17ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [17ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [35ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [36ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [38ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [38ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [75ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [77ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [78ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [92ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [95ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1937552651(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@4a344a39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811) -> (continuation of Deferred@1896327399 after retry RPC@238995604), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@409c8882 -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811) -> (continuation of Deferred@1896327399 after retry RPC@238995604))) after lookup
00:24:18.479 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:18.480 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.480 [DEBUG - kudu-nio-4] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.480 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.937 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:18.937 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:18.937 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=555), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [17ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [17ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [35ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [36ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [38ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [38ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [75ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [77ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [78ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [92ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [95ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [97ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [98ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [98ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [552ms] refreshing cache from master, [552ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [554ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1937552651(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@4a344a39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811) -> (continuation of Deferred@1896327399 after retry RPC@238995604) -> (continuation of Deferred@1829418812 after retry RPC@352893025), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@409c8882 -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811) -> (continuation of Deferred@1896327399 after retry RPC@238995604) -> (continuation of Deferred@1829418812 after retry RPC@352893025))) after lookup
00:24:18.938 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:18.939 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.939 [DEBUG - kudu-nio-5] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:18.939 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.643456 26048 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.643643 26048 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.645135 26048 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:19.646473 25745 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:19.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:19.648643 25745 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:19.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.654562 25983 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.655061 25983 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 7.
00:24:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.656128 25744 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.656565 26048 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Leader pre-election won for term 9
00:24:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.656718 26048 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Starting leader election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.656845 26048 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 8 FOLLOWER]: Advancing to term 9
00:24:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.660923 26048 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:19.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.661823 26048 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 9 election: Requested vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.663642 25983 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.663897 25983 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 7 FOLLOWER]: Advancing to term 9
00:24:19.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:19.666275 25745 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.668310 25983 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 9.
00:24:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.668784 25744 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.669064 26048 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Leader election won for term 9
00:24:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.669552 26048 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 LEADER]: Becoming Leader. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Running, Role: LEADER
00:24:19.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.670097 26048 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:19.674819 24267 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 reported cstate change: term changed from 8 to 9, leader changed from af84f760f77249aa81d9789e1b93e381 (127.23.168.129) to 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130). New cstate: current_term: 9 leader_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.156309 25983 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Refusing update from remote peer 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
00:24:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.157199 26048 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
00:24:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.173817 25745 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 -> Peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Couldn't send request to peer af84f760f77249aa81d9789e1b93e381. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:20.537 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:20.537 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:20.537 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2155), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [15ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [17ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [17ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [35ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [36ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [38ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [38ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [75ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [76ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [77ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [78ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [92ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [95ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [97ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [98ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [98ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [552ms] refreshing cache from master, [552ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [554ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [556ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [557ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [557ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [2152ms] refreshing cache from master, [2152ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [2154ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1937552651(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@4a344a39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811) -> (continuation of Deferred@1896327399 after retry RPC@238995604) -> (continuation of Deferred@1829418812 after retry RPC@352893025) -> (continuation of Deferred@1333392704 after retry RPC@526550910), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@409c8882 -> wakeup thread Time-limited test -> (continuation of Deferred@1941847829 after retry RPC@1472356728) -> (continuation of Deferred@674515289 after retry RPC@1413832811) -> (continuation of Deferred@1896327399 after retry RPC@238995604) -> (continuation of Deferred@1829418812 after retry RPC@352893025) -> (continuation of Deferred@1333392704 after retry RPC@526550910))) after lookup
00:24:20.538 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:20.539 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] Successfully connected to peer
00:24:20.564 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xa2962c6c, L:/127.0.0.1:48940 - R:/127.23.168.130:34095] using TOKEN/null
00:24:20.585 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.129:40013
00:24:20.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.586830 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.910801 26079 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.911202 26079 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.911325 26079 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.919930 26079 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.920161 26079 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.129
00:24:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.930343 26079 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:20.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:20.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:20.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.932454 26079 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:20.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.934392 26079 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.943367 26087 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.943650 26086 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:20.944705 26089 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.945142 26079 server_base.cc:1061] running on GCE node
00:24:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.945678 26079 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.946672 26079 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:20.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.947891 26079 hybrid_clock.cc:648] HybridClock initialized: now 1783470260947849 us; error 34 us; skew 500 ppm
00:24:20.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.948238 26079 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.950409 26079 webserver.cc:534] Webserver started at http://127.23.168.129:32839/ using document root <none> and password file <none>
00:24:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.951144 26079 fs_manager.cc:362] Metadata directory not provided
00:24:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.951277 26079 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.956655 26079 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.002s
00:24:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.959403 26095 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.960702 26079 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.003s
00:24:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.960873 26079 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.961606 26079 fs_report.cc:389] FS layout report
00:24:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/data
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.984766 26079 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.985653 26079 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.986003 26079 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.986977 26079 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:20.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.989020 26102 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.994714 26079 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.994813 26079 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.994942 26079 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.997859 26079 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.998000 26079 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
00:24:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:20.998520 26102 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap starting.
00:24:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.030361 26102 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Log is configured to *not* fsync() on all Append() calls
00:24:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.053797 26079 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.129:40013
00:24:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.053870 26209 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.129:40013 every 8 connection(s)
00:24:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.056813 26079 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.063875 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 26079
00:24:21.064 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:21.105 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:21.106 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.130:34095
00:24:21.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:24:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.107324 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 25727
00:24:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.107555 26102 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:21.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.108387 26102 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap complete.
00:24:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.124001 26210 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.124475 26210 heartbeater.cc:461] Registering TS with master...
00:24:21.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.125803 26210 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.129626 24411 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.142721 26102 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent bootstrapping tablet: real 0.145s	user 0.050s	sys 0.030s
00:24:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.143136 26211 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.143308 26211 heartbeater.cc:461] Registering TS with master...
00:24:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.143914 26211 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.145737 24268 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.147456 24268 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.129:37451
00:24:21.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.150394 25852 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.151183 26215 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.151348 26215 heartbeater.cc:461] Registering TS with master...
00:24:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.151670 26215 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.153225 24341 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.158684 26102 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.159669 26102 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Initialized, Role: FOLLOWER
00:24:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.160475 26102 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:21.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.162405 26102 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent starting tablet: real 0.017s	user 0.008s	sys 0.002s
00:24:21.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.163244 26211 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:21.270349 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 25727
00:24:21.295 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] handling channelInactive
00:24:21.295 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state READY due to: connection closed
00:24:21.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:21.294485 24327 connection.cc:573] server connection from 127.23.168.130:45263 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:21.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:21.295032 24397 connection.cc:573] server connection from 127.23.168.130:34957 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:21.295 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:21.296 [DEBUG - Time-limited test] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:21.297 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.297 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.297 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.317 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:21.317 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:21.318 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [1ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1946558583(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@7fd400fe -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d866680 -> wakeup thread Time-limited test)) after lookup
00:24:21.318 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:21.320 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.320 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:21.337 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:21.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [1ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [22ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [24ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1946558583(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@7fd400fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d866680 -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599))) after lookup
00:24:21.338 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:21.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.339 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:21.357 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:21.358 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [1ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [22ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [24ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [41ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [43ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1946558583(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@7fd400fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d866680 -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509))) after lookup
00:24:21.358 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:21.359 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.359 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.359 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.477 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:21.477 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:21.477 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=181), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [1ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [22ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [24ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [41ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [43ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [62ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [63ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [63ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [180ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1946558583(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@7fd400fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509) -> (continuation of Deferred@257148944 after retry RPC@1746496996), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d866680 -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509) -> (continuation of Deferred@257148944 after retry RPC@1746496996))) after lookup
00:24:21.478 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:21.479 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.479 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.479 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:21.797 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:21.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=501), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [1ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [22ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [24ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [41ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [43ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [62ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [63ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [63ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [180ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [182ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [183ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [183ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [498ms] refreshing cache from master, [498ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [500ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1946558583(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@7fd400fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509) -> (continuation of Deferred@257148944 after retry RPC@1746496996) -> (continuation of Deferred@1119051003 after retry RPC@1634107140), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d866680 -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509) -> (continuation of Deferred@257148944 after retry RPC@1746496996) -> (continuation of Deferred@1119051003 after retry RPC@1634107140))) after lookup
00:24:21.798 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:21.799 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.799 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:21.799 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095
00:24:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.458324 26225 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.458581 26225 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.460448 26225 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:22.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:22.461746 26099 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:22.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:22.463357 26099 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.468112 25983 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.468899 26097 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.469328 26225 raft_consensus.cc:2749] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
00:24:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.765232 26231 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 9e8a7a02fccc4f5bb47477a8c99ae1e8)
00:24:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.765416 26231 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.766908 26231 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:22.768194 25918 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:22.770017 25918 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.775214 26164 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381" is_pre_election: true
00:24:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.775573 26164 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 8.
00:24:22.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.776480 25917 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.776872 26231 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Leader pre-election won for term 10
00:24:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.777011 26231 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 9e8a7a02fccc4f5bb47477a8c99ae1e8)
00:24:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.777115 26231 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 9 FOLLOWER]: Advancing to term 10
00:24:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.779381 26231 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.780026 26231 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 10 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.780604 26164 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "81d9cd709bc146289a569ff26e47b3c8" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.780820 26164 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 8 FOLLOWER]: Advancing to term 10
00:24:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:22.781445 25918 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111)
00:24:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.784888 26164 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 81d9cd709bc146289a569ff26e47b3c8 in term 10.
00:24:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.785334 25917 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381; no voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8
00:24:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.785652 26231 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 FOLLOWER]: Leader election won for term 10
00:24:22.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.786124 26231 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 LEADER]: Becoming Leader. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Running, Role: LEADER
00:24:22.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.786729 26231 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:22.790653 24268 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 reported cstate change: term changed from 9 to 10, leader changed from 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130) to 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131). New cstate: current_term: 10 leader_uuid: "81d9cd709bc146289a569ff26e47b3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: HEALTHY } } }
00:24:22.997 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:22.997 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:22.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1702), Traces: [0ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [1ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [22ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [24ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [41ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [43ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [62ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [63ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [63ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [180ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [182ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [183ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [183ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [498ms] refreshing cache from master, [498ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [500ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [502ms] sending RPC to server 9e8a7a02fccc4f5bb47477a8c99ae1e8, [503ms] received response from server 9e8a7a02fccc4f5bb47477a8c99ae1e8: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [503ms] delaying RPC due to: Network error: Failed to connect to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095): Connection refused: /127.23.168.130:34095, [1698ms] refreshing cache from master, [1698ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1701ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1946558583(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@7fd400fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509) -> (continuation of Deferred@257148944 after retry RPC@1746496996) -> (continuation of Deferred@1119051003 after retry RPC@1634107140) -> (continuation of Deferred@2029917284 after retry RPC@707971616), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d866680 -> wakeup thread Time-limited test -> (continuation of Deferred@1535183977 after retry RPC@1973273599) -> (continuation of Deferred@1736682929 after retry RPC@2090469509) -> (continuation of Deferred@257148944 after retry RPC@1746496996) -> (continuation of Deferred@1119051003 after retry RPC@1634107140) -> (continuation of Deferred@2029917284 after retry RPC@707971616))) after lookup
00:24:22.998 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:22.999 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] Successfully connected to peer
00:24:23.023 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x9a977b2d, L:/127.0.0.1:36544 - R:/127.23.168.131:37089] using TOKEN/null
00:24:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.028681 26164 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Refusing update from remote peer 81d9cd709bc146289a569ff26e47b3c8: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 34. (index mismatch)
00:24:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.028769 25918 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 -> Peer 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095): Couldn't send request to peer 9e8a7a02fccc4f5bb47477a8c99ae1e8. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.130:34095: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.029549 26236 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
00:24:23.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.038501 26249 mvcc.cc:204] Tried to move back new op lower bound from 7305094197355737088 to 7305094196378071040. Current Snapshot: MvccSnapshot[applied={T|T < 7305094197355737088}]
00:24:23.059 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.130:34095
00:24:23.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.060567 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:34095
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36067
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.387104 26258 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.387380 26258 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.387472 26258 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.395462 26258 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.395612 26258 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.130
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.406276 26258 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.130:34095
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.130
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36067
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.130
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/logs
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.408217 26258 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.410446 26258 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.421201 26263 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.422241 26264 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:23.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.425132 26266 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:23.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.425228 26258 server_base.cc:1061] running on GCE node
00:24:23.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.426162 26258 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.429132 26258 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.430339 26258 hybrid_clock.cc:648] HybridClock initialized: now 1783470263430284 us; error 49 us; skew 500 ppm
00:24:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.430605 26258 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.432646 26258 webserver.cc:534] Webserver started at http://127.23.168.130:36067/ using document root <none> and password file <none>
00:24:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.433296 26258 fs_manager.cc:362] Metadata directory not provided
00:24:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.433382 26258 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:23.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.438624 26258 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.002s
00:24:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.441146 26272 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.442463 26258 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.003s
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.442660 26258 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.443351 26258 fs_report.cc:389] FS layout report
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/wal
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/data
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.476902 26258 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:23.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.477922 26258 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:23.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.478323 26258 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.479372 26258 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.481809 26279 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.487638 26258 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.487720 26258 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.487819 26258 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.490772 26258 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.490859 26258 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.002s	sys 0.000s
00:24:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.491356 26279 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap starting.
00:24:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.519867 26279 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log is configured to *not* fsync() on all Append() calls
00:24:23.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.537043 26258 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.130:34095
00:24:23.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.537124 26386 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.130:34095 every 8 connection(s)
00:24:23.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.538561 26258 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-1/data/info.pb
00:24:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.547451 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 26258
00:24:23.548 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:23.584 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.583220 26387 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.583638 26387 heartbeater.cc:461] Registering TS with master...
00:24:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.584676 26387 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.584911 26393 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.585032 26393 heartbeater.cc:461] Registering TS with master...
00:24:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.585404 26393 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:23.587 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.131:37089
00:24:23.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:24:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.587612 24341 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:23.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.587767 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 25900
00:24:23.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.588495 26279 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:23.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.589170 26279 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Bootstrap complete.
00:24:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.590039 24412 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.590619 26279 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent bootstrapping tablet: real 0.100s	user 0.053s	sys 0.019s
00:24:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.595597 26388 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.595744 26388 heartbeater.cc:461] Registering TS with master...
00:24:23.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.596233 26388 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:23.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.597487 24268 ts_manager.cc:194] Re-registered known tserver with Master: 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095)
00:24:23.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.597508 26279 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.598726 26279 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Initialized, Role: FOLLOWER
00:24:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.599011 24268 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.130:40305
00:24:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.599748 26279 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.603327 26279 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8: Time spent starting tablet: real 0.013s	user 0.008s	sys 0.004s
00:24:23.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.604067 26388 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:23.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.609356 26024 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:23.705440 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 25900
00:24:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:23.722910 26096 connection.cc:573] server connection from 127.23.168.131:50079 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:23.723 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] handling channelInactive
00:24:23.723 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state READY due to: connection closed
00:24:23.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:23.724 [DEBUG - Time-limited test] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:23.725 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.725 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.726 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.736 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:23.737 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:23.737 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1521982837(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@66e83fc9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@31d9adfc -> wakeup thread Time-limited test)) after lookup
00:24:23.738 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:23.738 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.739 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.739 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.757 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:23.757 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:23.758 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [15ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [15ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1521982837(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@66e83fc9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@31d9adfc -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632))) after lookup
00:24:23.758 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:23.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.759 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.796 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:23.797 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:23.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [15ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [15ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [35ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [35ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1521982837(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@66e83fc9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@31d9adfc -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116))) after lookup
00:24:23.797 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:23.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.799 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:23.897 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:23.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [15ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [15ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [35ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [35ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [75ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [75ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [172ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1521982837(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@66e83fc9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116) -> (continuation of Deferred@2139622236 after retry RPC@182393931), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@31d9adfc -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116) -> (continuation of Deferred@2139622236 after retry RPC@182393931))) after lookup
00:24:23.898 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:23.899 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.899 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:23.899 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:24.297 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:24.297 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)[L],9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:24.298 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=573), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [15ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [15ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [35ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [35ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [75ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [75ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [172ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [174ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [175ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [175ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [570ms] refreshing cache from master, [570ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [573ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1521982837(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@66e83fc9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116) -> (continuation of Deferred@2139622236 after retry RPC@182393931) -> (continuation of Deferred@863291557 after retry RPC@2004023594), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@31d9adfc -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116) -> (continuation of Deferred@2139622236 after retry RPC@182393931) -> (continuation of Deferred@863291557 after retry RPC@2004023594))) after lookup
00:24:24.298 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] connecting to peer
00:24:24.302 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:24.302 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] cleaning up while in state CONNECTING due to: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:24.302 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089
00:24:24.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:24.785085 26402 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:24.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:24.785346 26402 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:24.786933 26402 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:24.791186 26274 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:24.793035 26274 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:24.796723 26164 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "af84f760f77249aa81d9789e1b93e381" is_pre_election: true
00:24:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:24.797483 26275 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: 81d9cd709bc146289a569ff26e47b3c8, af84f760f77249aa81d9789e1b93e381
00:24:24.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:24.797842 26402 raft_consensus.cc:2749] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
00:24:25.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.207091 26407 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 81d9cd709bc146289a569ff26e47b3c8)
00:24:25.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.207432 26407 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.208345 26407 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.210969 26097 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.216516 26341 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" is_pre_election: true
00:24:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.216952 26341 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 9.
00:24:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.217880 26099 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8
00:24:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.218215 26407 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Leader pre-election won for term 11
00:24:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.218361 26407 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 81d9cd709bc146289a569ff26e47b3c8)
00:24:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.218447 26407 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 10 FOLLOWER]: Advancing to term 11
00:24:25.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.220422 26407 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.221084 26407 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 11 election: Requested vote from peers 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130:34095), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.221458 26341 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "af84f760f77249aa81d9789e1b93e381" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8"
00:24:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.221659 26341 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 9 FOLLOWER]: Advancing to term 11
00:24:25.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.223402 26097 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111)
00:24:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.225955 26341 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af84f760f77249aa81d9789e1b93e381 in term 11.
00:24:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.226483 26099 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9e8a7a02fccc4f5bb47477a8c99ae1e8, af84f760f77249aa81d9789e1b93e381; no voters: 81d9cd709bc146289a569ff26e47b3c8
00:24:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.226804 26407 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 11 FOLLOWER]: Leader election won for term 11
00:24:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.227258 26407 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 11 LEADER]: Becoming Leader. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Running, Role: LEADER
00:24:25.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.227788 26407 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.231824 24268 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 reported cstate change: term changed from 10 to 11, leader changed from 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131) to af84f760f77249aa81d9789e1b93e381 (127.23.168.129). New cstate: current_term: 11 leader_uuid: "af84f760f77249aa81d9789e1b93e381" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:25.417 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:25.417 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:25.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1693), Traces: [0ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [1ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [2ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [15ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [15ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [35ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [35ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [75ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [75ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [172ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [174ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [175ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [175ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [570ms] refreshing cache from master, [570ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [573ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [574ms] sending RPC to server 81d9cd709bc146289a569ff26e47b3c8, [578ms] received response from server 81d9cd709bc146289a569ff26e47b3c8: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [578ms] delaying RPC due to: Network error: Failed to connect to peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089): Connection refused: /127.23.168.131:37089, [1690ms] refreshing cache from master, [1690ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1693ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1521982837(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@66e83fc9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116) -> (continuation of Deferred@2139622236 after retry RPC@182393931) -> (continuation of Deferred@863291557 after retry RPC@2004023594) -> (continuation of Deferred@1809166278 after retry RPC@193896511), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@31d9adfc -> wakeup thread Time-limited test -> (continuation of Deferred@889624737 after retry RPC@1229747632) -> (continuation of Deferred@2109686078 after retry RPC@1467239116) -> (continuation of Deferred@2139622236 after retry RPC@182393931) -> (continuation of Deferred@863291557 after retry RPC@2004023594) -> (continuation of Deferred@1809166278 after retry RPC@193896511))) after lookup
00:24:25.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:25.420 [DEBUG - kudu-nio-4] (Connection.java:757) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] Successfully connected to peer
00:24:25.443 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x322eb694, L:/127.0.0.1:39746 - R:/127.23.168.129:40013] using TOKEN/null
00:24:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.449234 26341 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Refusing update from remote peer af84f760f77249aa81d9789e1b93e381: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 38. (index mismatch)
00:24:25.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.450191 26412 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
00:24:25.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.450533 26097 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 -> Peer 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089): Couldn't send request to peer 81d9cd709bc146289a569ff26e47b3c8. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.131:37089: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:25.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.461077 26425 mvcc.cc:204] Tried to move back new op lower bound from 7305094207269658624 to 7305094206377418752. Current Snapshot: MvccSnapshot[applied={T|T < 7305094207269658624}]
00:24:25.479 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.131:37089
00:24:25.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.480003 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:37089
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32819
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.807974 26434 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.808261 26434 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.808346 26434 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.816454 26434 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.816604 26434 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.131
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.826354 26434 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.131:37089
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.131
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32819
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.131
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/logs
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.827817 26434 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:25.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.829497 26434 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.838917 26440 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.839021 26439 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:25.839772 26442 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.840039 26434 server_base.cc:1061] running on GCE node
00:24:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.840631 26434 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:25.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.841559 26434 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:25.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.842783 26434 hybrid_clock.cc:648] HybridClock initialized: now 1783470265842732 us; error 36 us; skew 500 ppm
00:24:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.843151 26434 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:25.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.845506 26434 webserver.cc:534] Webserver started at http://127.23.168.131:32819/ using document root <none> and password file <none>
00:24:25.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.846390 26434 fs_manager.cc:362] Metadata directory not provided
00:24:25.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.846524 26434 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:25.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.851899 26434 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.002s
00:24:25.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.854772 26448 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.856370 26434 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.856549 26434 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:57 on dist-test-slave-4ccz"
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.857283 26434 fs_report.cc:389] FS layout report
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/wal
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/data
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.888240 26434 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:25.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.889202 26434 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:25.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.889557 26434 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:25.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.890591 26434 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:25.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.892621 26455 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.898435 26434 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.898545 26434 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.898684 26434 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.901638 26434 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.901759 26434 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
00:24:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.902135 26455 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap starting.
00:24:25.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.930577 26455 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Log is configured to *not* fsync() on all Append() calls
00:24:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.965811 26434 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.131:37089
00:24:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.965878 26564 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.131:37089 every 8 connection(s)
00:24:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.967556 26434 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-2/data/info.pb
00:24:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.968819 26455 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:25.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.969635 26455 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Bootstrap complete.
00:24:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.971134 26455 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent bootstrapping tablet: real 0.069s	user 0.049s	sys 0.018s
00:24:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.977735 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 26434
00:24:25.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:25.996 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:25.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.996079 26455 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:25.997 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.168.129:40013
00:24:25.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.997084 26455 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81d9cd709bc146289a569ff26e47b3c8, State: Initialized, Role: FOLLOWER
00:24:25.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.999253 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 26079
00:24:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:25.999214 26455 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.011845 26565 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.012231 26565 heartbeater.cc:461] Registering TS with master...
00:24:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.012952 26455 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8: Time spent starting tablet: real 0.042s	user 0.010s	sys 0.002s
00:24:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.013427 26565 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:26.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.016956 24412 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.020447 26566 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.020581 26566 heartbeater.cc:461] Registering TS with master...
00:24:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.021091 26566 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.022403 24267 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.023345 26567 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.023481 26567 heartbeater.cc:461] Registering TS with master...
00:24:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.023893 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.131:58413
00:24:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.023939 26567 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:26.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.025686 26205 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:26.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.025678 24340 ts_manager.cc:194] Re-registered known tserver with Master: 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.026856 26566 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:26.124934 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 26079
00:24:26.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:26.142702 26274 connection.cc:573] server connection from 127.23.168.129:36633 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:26.143 [DEBUG - kudu-nio-4] (Connection.java:239) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] handling channelInactive
00:24:26.143 [DEBUG - kudu-nio-4] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state READY due to: connection closed
00:24:26.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:26.144 [DEBUG - Time-limited test] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:26.145 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.145 [DEBUG - kudu-nio-5] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.145 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.156 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:26.157 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:26.157 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test)) after lookup
00:24:26.157 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:26.158 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.158 [DEBUG - kudu-nio-6] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.158 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.177 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:26.177 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:26.177 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996))) after lookup
00:24:26.178 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:26.179 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.179 [DEBUG - kudu-nio-7] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.179 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.216 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:26.217 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:26.217 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340))) after lookup
00:24:26.217 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:26.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.218 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.237 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:26.237 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:26.237 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=93), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [93ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633))) after lookup
00:24:26.238 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:26.239 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.239 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:26.358 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:26.358 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=214), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [93ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [94ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [95ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [95ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [210ms] refreshing cache from master, [211ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [213ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633) -> (continuation of Deferred@74329393 after retry RPC@1166356413), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633) -> (continuation of Deferred@74329393 after retry RPC@1166356413))) after lookup
00:24:26.358 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:26.359 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.359 [DEBUG - kudu-nio-2] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:26.360 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.626356 26580 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.626555 26580 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:27.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.627223 26580 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:27.630266 26275 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.637742 26518 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8" is_pre_election: true
00:24:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.638348 26518 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 10.
00:24:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.639583 26274 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:27.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.640074 26580 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Leader pre-election won for term 12
00:24:27.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.640247 26580 raft_consensus.cc:493] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Starting leader election (detected failure of leader af84f760f77249aa81d9789e1b93e381)
00:24:27.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.640328 26580 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 11 FOLLOWER]: Advancing to term 12
00:24:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.642693 26580 raft_consensus.cc:515] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.643252 26580 leader_election.cc:290] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 12 election: Requested vote from peers af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013), 81d9cd709bc146289a569ff26e47b3c8 (127.23.168.131:37089)
00:24:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.643810 26518 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c17cfea4b88143bc98ad44ebf3f798ac" candidate_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "81d9cd709bc146289a569ff26e47b3c8"
00:24:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.644007 26518 raft_consensus.cc:3060] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 10 FOLLOWER]: Advancing to term 12
00:24:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:27.644829 26275 leader_election.cc:336] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111)
00:24:27.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.647796 26518 raft_consensus.cc:2468] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e8a7a02fccc4f5bb47477a8c99ae1e8 in term 12.
00:24:27.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.648231 26274 leader_election.cc:304] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 81d9cd709bc146289a569ff26e47b3c8, 9e8a7a02fccc4f5bb47477a8c99ae1e8; no voters: af84f760f77249aa81d9789e1b93e381
00:24:27.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.648556 26580 raft_consensus.cc:2804] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 12 FOLLOWER]: Leader election won for term 12
00:24:27.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.648916 26580 raft_consensus.cc:697] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [term 12 LEADER]: Becoming Leader. State: Replica: 9e8a7a02fccc4f5bb47477a8c99ae1e8, State: Running, Role: LEADER
00:24:27.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.649564 26580 consensus_queue.cc:237] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:27.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:27.653146 24267 catalog_manager.cc:5696] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 reported cstate change: term changed from 11 to 12, leader changed from af84f760f77249aa81d9789e1b93e381 (127.23.168.129) to 9e8a7a02fccc4f5bb47477a8c99ae1e8 (127.23.168.130). New cstate: current_term: 12 leader_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } health_report { overall_health: UNKNOWN } } }
00:24:27.660 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:27.660 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095),af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)[L]]}]
00:24:27.661 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1516), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [93ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [94ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [95ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [95ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [210ms] refreshing cache from master, [211ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [213ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [214ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [215ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [216ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1510ms] refreshing cache from master, [1510ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1516ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633) -> (continuation of Deferred@74329393 after retry RPC@1166356413) -> (continuation of Deferred@1466831337 after retry RPC@718903464), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633) -> (continuation of Deferred@74329393 after retry RPC@1166356413) -> (continuation of Deferred@1466831337 after retry RPC@718903464))) after lookup
00:24:27.661 [DEBUG - kudu-nio-2] (Connection.java:748) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] connecting to peer
00:24:27.663 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:27.663 [DEBUG - kudu-nio-3] (Connection.java:709) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] cleaning up while in state CONNECTING due to: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:27.663 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013) for tablet c17cfea4b88143bc98ad44ebf3f798ac: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013
00:24:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:28.047077 26518 raft_consensus.cc:1275] T c17cfea4b88143bc98ad44ebf3f798ac P 81d9cd709bc146289a569ff26e47b3c8 [term 12 FOLLOWER]: Refusing update from remote peer 9e8a7a02fccc4f5bb47477a8c99ae1e8: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
00:24:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:28.047960 26580 consensus_queue.cc:1048] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
00:24:28.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:28.062193 26275 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 -> Peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Couldn't send request to peer af84f760f77249aa81d9789e1b93e381. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:29.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:29.950392 26275 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111) [suppressed 6 similar messages]
00:24:30.557 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c17cfea4b88143bc98ad44ebf3f798ac for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778' with partition [<start>, <end>)
00:24:30.557 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c17cfea4b88143bc98ad44ebf3f798ac@[81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089),9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)[L],af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)]}]
00:24:30.558 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c17cfea4b88143bc98ad44ebf3f798ac, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4413), Traces: [0ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [13ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [14ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [14ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [34ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [35ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [35ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [73ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [74ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [74ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [93ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [94ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [95ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [95ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [210ms] refreshing cache from master, [211ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [213ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [214ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [215ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [216ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1510ms] refreshing cache from master, [1510ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [1516ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, [1517ms] sending RPC to server af84f760f77249aa81d9789e1b93e381, [1519ms] received response from server af84f760f77249aa81d9789e1b93e381: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [1519ms] delaying RPC due to: Network error: Failed to connect to peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013): Connection refused: /127.23.168.129:40013, [4410ms] refreshing cache from master, [4410ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.168.190:39191, [4413ms] Sub RPC GetTableLocations: received response from server master-127.23.168.190:39191: OK, deferred=Deferred@1557432703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$230/0x00007f336036b560@22254582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633) -> (continuation of Deferred@74329393 after retry RPC@1166356413) -> (continuation of Deferred@1466831337 after retry RPC@718903464) -> (continuation of Deferred@652535019 after retry RPC@1655232082), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77c9b1cc -> wakeup thread Time-limited test -> (continuation of Deferred@1353903671 after retry RPC@1791975996) -> (continuation of Deferred@1790644221 after retry RPC@2079608340) -> (continuation of Deferred@1207341629 after retry RPC@514584633) -> (continuation of Deferred@74329393 after retry RPC@1166356413) -> (continuation of Deferred@1466831337 after retry RPC@718903464) -> (continuation of Deferred@652535019 after retry RPC@1655232082))) after lookup
00:24:30.558 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] connecting to peer
00:24:30.559 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] Successfully connected to peer
00:24:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.567840 26275 consensus_peers.cc:597] T c17cfea4b88143bc98ad44ebf3f798ac P 9e8a7a02fccc4f5bb47477a8c99ae1e8 -> Peer af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013): Couldn't send request to peer af84f760f77249aa81d9789e1b93e381. Status: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:24:30.577 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0xf4471a2b, L:/127.0.0.1:42766 - R:/127.23.168.130:34095] using TOKEN/null
00:24:30.598 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.168.129:40013
00:24:30.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.599493 24226 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4SKNut/build/asan/bin/kudu
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.918174 26621 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:24:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.918531 26621 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.918651 26621 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:24:30.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.926719 26621 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:24:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.926944 26621 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.168.129
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.936866 26621 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.168.148:46127
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.168.129:40013
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.168.129
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32839
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.168.190:39191,127.23.168.189:38643,127.23.168.188:44491
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.168.129
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/logs
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on bdcb31816ec0
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16842
00:24:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.938933 26621 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:30.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.940805 26621 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:24:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.950307 26627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.950800 26626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:30.951195 26629 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:30.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.951881 26621 server_base.cc:1061] running on GCE node
00:24:30.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.952370 26621 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:30.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.953169 26621 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:30.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.954397 26621 hybrid_clock.cc:648] HybridClock initialized: now 1783470270954326 us; error 56 us; skew 500 ppm
00:24:30.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.954718 26621 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:30.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.956885 26621 webserver.cc:534] Webserver started at http://127.23.168.129:32839/ using document root <none> and password file <none>
00:24:30.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.957647 26621 fs_manager.cc:362] Metadata directory not provided
00:24:30.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.957767 26621 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.963198 26621 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.001s
00:24:30.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.965903 26635 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.967267 26621 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.967449 26621 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data,/tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af84f760f77249aa81d9789e1b93e381"
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:23:56 on dist-test-slave-4ccz"
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.968134 26621 fs_report.cc:389] FS layout report
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/wal
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/data
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.987035 26621 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:24:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.987910 26621 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.988236 26621 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:30.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.989192 26621 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:24:30.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.991281 26642 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:24:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.997046 26621 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:24:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.997157 26621 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:24:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:30.997290 26621 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:24:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.000259 26621 ts_tablet_manager.cc:616] Registered 1 tablets
00:24:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.000379 26621 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
00:24:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.000727 26642 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap starting.
00:24:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.030664 26642 log.cc:826] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Log is configured to *not* fsync() on all Append() calls
00:24:31.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.057298 26621 rpc_server.cc:307] RPC server started. Bound to: 127.23.168.129:40013
00:24:31.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.057368 26749 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.168.129:40013 every 8 connection(s)
00:24:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.058774 26621 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4SKNut/test-tmp/mini-kudu-cluster9746841617994720985/ts-0/data/info.pb
00:24:31.066 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.065645 24226 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4SKNut/build/asan/bin/kudu as pid 26621
00:24:31.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.076344 26642 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:24:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.077384 26642 tablet_bootstrap.cc:492] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Bootstrap complete.
00:24:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.079417 26642 ts_tablet_manager.cc:1403] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent bootstrapping tablet: real 0.079s	user 0.054s	sys 0.020s
00:24:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.088414 26642 raft_consensus.cc:359] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.089593 26642 raft_consensus.cc:740] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: af84f760f77249aa81d9789e1b93e381, State: Initialized, Role: FOLLOWER
00:24:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.090478 26642 consensus_queue.cc:260] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8a7a02fccc4f5bb47477a8c99ae1e8" member_type: VOTER last_known_addr { host: "127.23.168.130" port: 34095 } } peers { permanent_uuid: "af84f760f77249aa81d9789e1b93e381" member_type: VOTER last_known_addr { host: "127.23.168.129" port: 40013 } } peers { permanent_uuid: "81d9cd709bc146289a569ff26e47b3c8" member_type: VOTER last_known_addr { host: "127.23.168.131" port: 37089 } }
00:24:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.121735 26750 heartbeater.cc:344] Connected to a master server at 127.23.168.188:44491
00:24:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.122270 26750 heartbeater.cc:461] Registering TS with master...
00:24:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.123692 26750 heartbeater.cc:507] Master 127.23.168.188:44491 requested a full tablet report, sending...
00:24:31.128 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:31.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.127208 24412 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:31.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.130559 26752 heartbeater.cc:344] Connected to a master server at 127.23.168.189:38643
00:24:31.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.130755 26752 heartbeater.cc:461] Registering TS with master...
00:24:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.131469 26752 heartbeater.cc:507] Master 127.23.168.189:38643 requested a full tablet report, sending...
00:24:31.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.135574 26642 ts_tablet_manager.cc:1434] T c17cfea4b88143bc98ad44ebf3f798ac P af84f760f77249aa81d9789e1b93e381: Time spent starting tablet: real 0.056s	user 0.028s	sys 0.027s
00:24:31.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.136400 26751 heartbeater.cc:344] Connected to a master server at 127.23.168.190:39191
00:24:31.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.136548 26751 heartbeater.cc:461] Registering TS with master...
00:24:31.136 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470237778, tablet=c17cfea4b88143bc98ad44ebf3f798ac, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.136864 24340 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.137048 26751 heartbeater.cc:507] Master 127.23.168.190:39191 requested a full tablet report, sending...
00:24:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.139748 24267 ts_manager.cc:194] Re-registered known tserver with Master: af84f760f77249aa81d9789e1b93e381 (127.23.168.129:40013)
00:24:31.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.141356 24267 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.168.129:49319
00:24:31.142 [DEBUG - main] (Connection.java:575) [peer af84f760f77249aa81d9789e1b93e381(127.23.168.129:40013)] disconnecting while in state TERMINATED
00:24:31.143 [DEBUG - main] (Connection.java:575) [peer master-127.23.168.189:38643(127.23.168.189:38643)] disconnecting while in state READY
00:24:31.143 [DEBUG - main] (Connection.java:575) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] disconnecting while in state READY
00:24:31.143 [DEBUG - main] (Connection.java:575) [peer 81d9cd709bc146289a569ff26e47b3c8(127.23.168.131:37089)] disconnecting while in state TERMINATED
00:24:31.144 [DEBUG - main] (Connection.java:575) [peer master-127.23.168.190:39191(127.23.168.190:39191)] disconnecting while in state READY
00:24:31.144 [DEBUG - main] (Connection.java:575) [peer master-127.23.168.188:44491(127.23.168.188:44491)] disconnecting while in state READY
00:24:31.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.143983 26751 heartbeater.cc:499] Master 127.23.168.190:39191 was elected leader, sending a full tablet report...
00:24:31.146 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.23.168.190:39191(127.23.168.190:39191)] handling channelInactive
00:24:31.146 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] handling channelInactive
00:24:31.146 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.168.188:44491(127.23.168.188:44491)] handling channelInactive
00:24:31.146 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.168.190:39191(127.23.168.190:39191)] cleaning up while in state READY due to: connection closed
00:24:31.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.168.188:44491(127.23.168.188:44491)] cleaning up while in state READY due to: connection closed
00:24:31.146 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e8a7a02fccc4f5bb47477a8c99ae1e8(127.23.168.130:34095)] cleaning up while in state READY due to: connection closed
00:24:31.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:24:31.147 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.168.189:38643(127.23.168.189:38643)] handling channelInactive
00:24:31.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.168.189:38643(127.23.168.189:38643)] cleaning up while in state READY due to: connection closed
00:24:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.148551 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 26621
00:24:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.158982 26745 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.250250 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 26621
00:24:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.267257 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 26258
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:31.268817 26767 negotiation.cc:336] Failed RPC negotiation. Trace:
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.171445 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.23.168.129:40013 (local address 127.23.168.130:59901)
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.172004 (+   559us) negotiation.cc:107] Waiting for socket to connect
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.172030 (+    26us) client_negotiation.cc:175] Beginning negotiation
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.172157 (+   127us) client_negotiation.cc:253] Sending NEGOTIATE NegotiatePB request
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.249662 (+ 77505us) client_negotiation.cc:272] Received NEGOTIATE NegotiatePB response
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.249669 (+     7us) client_negotiation.cc:395] Received NEGOTIATE response from server
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.249713 (+    44us) client_negotiation.cc:190] Negotiated authn=CERTIFICATE
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.252738 (+  3025us) client_negotiation.cc:515] Sending TLS_HANDSHAKE message to server
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.252766 (+    28us) client_negotiation.cc:253] Sending TLS_HANDSHAKE NegotiatePB request
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:24:31.267703 (+ 14937us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.23.168.129:40013: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":427,"thread_start_us":251,"threads_started":1}
00:24:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.276077 26382 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:31.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.388827 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 26258
00:24:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.407809 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 26434
00:24:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.415841 26560 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.506745 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 26434
00:24:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.524574 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 24238
00:24:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.525770 24299 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:31.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.647716 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 24238
00:24:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:31.670222 24399 connection.cc:573] server connection from 127.0.0.1:35520 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:24:31.670281 24326 connection.cc:573] server connection from 127.0.0.1:35936 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:24:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.670692 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 24306
00:24:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.671761 24371 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.785679 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 24306
00:24:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.807672 24226 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task4SKNut/build/asan/bin/kudu pid 24378
00:24:31.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.808777 24442 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:24:31.925494 24226 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4SKNut/build/asan/bin/kudu with pid 24378
00:24:31.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:24:31Z chronyd exiting

Time: 39.156

OK (1 test)

