JUnit version 4.13.2
.00:24:50.178 [INFO - main] (RandomUtils.java:49) Using random seed: 1786494290170
00:24:50.230 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
00:24:50.289 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:24:50.302 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLgeSWz/test-tmp
00:24:50.306 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
00:24:50.309 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLgeSWz/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
00:24:50.543 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

00:24:50.720 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:24:50.721 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

00:24:50.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:50.724749 23562 mini_oidc.cc:152] Starting JWKS server
00:24:50.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:50.727833 23562 webserver.cc:460] Webserver started at http://127.0.0.1:41133/ using document root <none> and password file <none>
00:24:50.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:50.728195 23562 mini_oidc.cc:189] Starting OIDC Discovery server
00:24:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:50.728993 23562 webserver.cc:460] Webserver started at http://127.0.0.1:42283/ using document root <none> and password file <none>
00:24:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:24:50Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
00:24:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:24:50Z Disabled control of system clock
00:24:50.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:50.763445 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/logs
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data/info.pb
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:24:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.190:44663
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.190
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:24:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41133/jwks.json with env {}
00:24:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.108603 23586 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:24:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.108934 23586 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.109021 23586 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:24:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.117172 23586 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:24:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.117308 23586 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:24:51.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.117350 23586 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:24:51.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.117398 23586 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:24:51.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.117435 23586 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:24:51.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.117483 23586 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41133/jwks.json
00:24:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.128547 23586 master_runner.cc:386] Master server non-default flags:
00:24:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data
00:24:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal
00:24:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:24:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:24:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.190:44663
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41133/jwks.json
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data/info.pb
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.190
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/logs
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:24:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:24:51.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:51.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.130082 23586 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:51.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.132428 23586 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:24:51.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.146135 23601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:51.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.146078 23594 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:51.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.146103 23593 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:51.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.147337 23586 server_base.cc:1034] running on GCE node
00:24:51.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.148404 23586 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.150144 23586 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.151419 23586 hybrid_clock.cc:648] HybridClock initialized: now 1786494291151337 us; error 49 us; skew 500 ppm
00:24:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.151901 23586 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.154523 23586 webserver.cc:460] Webserver started at http://127.23.2.190:42115/ using document root <none> and password file <none>
00:24:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.155369 23586 fs_manager.cc:359] Metadata directory not provided
00:24:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.155505 23586 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:51.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.155966 23586 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:24:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.158953 23586 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data/instance:
00:24:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9e0ab45df254654bb01e486a8f5765f"
00:24:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:51 on dist-test-slave-6bbx"
00:24:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.159978 23586 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal/instance:
00:24:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9e0ab45df254654bb01e486a8f5765f"
00:24:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:51 on dist-test-slave-6bbx"
00:24:51.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.165903 23586 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
00:24:51.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.170066 23612 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.171655 23586 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
00:24:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.171901 23586 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal
00:24:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9e0ab45df254654bb01e486a8f5765f"
00:24:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:51 on dist-test-slave-6bbx"
00:24:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.172156 23586 fs_report.cc:383] FS layout report
00:24:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal
00:24:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data/data
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:51.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:51.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:51.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:51.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.194017 23586 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:24:51.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.195111 23586 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:51.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.195487 23586 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.214152 23586 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.190:44663
00:24:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.214202 23722 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.190:44663 every 8 connection(s)
00:24:51.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.215811 23586 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/data/info.pb
00:24:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.220118 23723 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:24:51.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.222944 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 23586
00:24:51.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.223237 23562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-0/wal/instance
00:24:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.226322 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal
00:24:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data
00:24:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/logs
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data/info.pb
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.189:41391
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.189
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:51.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:24:51.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:51.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:24:51.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41133/jwks.json with env {}
00:24:51.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.229481 23723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } has no permanent_uuid. Determining permanent_uuid...
00:24:51.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.250422 23723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } has no permanent_uuid. Determining permanent_uuid...
00:24:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.251770 23613 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.189:41391: connect: Connection refused (error 111)
00:24:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.253562 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.189:41391: Network error: Client connection negotiation failed: client connection to 127.23.2.189:41391: connect: Connection refused (error 111)
00:24:51.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.302996 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } attempt: 1
00:24:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.305084 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.189:41391: Network error: Client connection negotiation failed: client connection to 127.23.2.189:41391: connect: Connection refused (error 111)
00:24:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.373417 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } attempt: 2
00:24:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.375742 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.189:41391: Network error: Client connection negotiation failed: client connection to 127.23.2.189:41391: connect: Connection refused (error 111)
00:24:51.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.467053 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } attempt: 3
00:24:51.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.470338 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.189:41391: Network error: Client connection negotiation failed: client connection to 127.23.2.189:41391: connect: Connection refused (error 111)
00:24:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.585988 23728 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:24:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.586391 23728 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.586524 23728 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:24:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.594452 23728 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:24:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.594604 23728 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:24:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.594676 23728 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:24:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.594748 23728 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:24:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.594801 23728 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:24:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.594870 23728 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41133/jwks.json
00:24:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.605139 23728 master_runner.cc:386] Master server non-default flags:
00:24:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.189:41391
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:24:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41133/jwks.json
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data/info.pb
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.189
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/logs
00:24:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:24:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:24:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:24:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:24:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:24:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.607133 23728 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.609957 23728 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:24:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.613669 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } attempt: 4
00:24:51.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.615887 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.189:41391: Network error: Client connection negotiation failed: client connection to 127.23.2.189:41391: connect: Connection refused (error 111)
00:24:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.622826 23743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.622860 23751 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.622888 23744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:51.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.623986 23728 server_base.cc:1034] running on GCE node
00:24:51.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.624943 23728 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.626578 23728 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.627789 23728 hybrid_clock.cc:648] HybridClock initialized: now 1786494291627719 us; error 35 us; skew 500 ppm
00:24:51.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.628193 23728 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.630838 23728 webserver.cc:460] Webserver started at http://127.23.2.189:32921/ using document root <none> and password file <none>
00:24:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.631719 23728 fs_manager.cc:359] Metadata directory not provided
00:24:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.631870 23728 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.632292 23728 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:24:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.635490 23728 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data/instance:
00:24:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f"
00:24:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:51 on dist-test-slave-6bbx"
00:24:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.636620 23728 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal/instance:
00:24:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f"
00:24:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:51 on dist-test-slave-6bbx"
00:24:51.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.642766 23728 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
00:24:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.647106 23761 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.648732 23728 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.648954 23728 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f"
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:51 on dist-test-slave-6bbx"
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.649214 23728 fs_report.cc:383] FS layout report
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data/data
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.668938 23728 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:24:51.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.670063 23728 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:51.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.670430 23728 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.689795 23728 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.189:41391
00:24:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.689850 23863 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.189:41391 every 8 connection(s)
00:24:51.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.691498 23728 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/data/info.pb
00:24:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.696153 23865 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:24:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.700596 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 23728
00:24:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.700843 23562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-1/wal/instance
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.702201 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/logs
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data/info.pb
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.188:46541
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.188
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:24:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:24:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41133/jwks.json with env {}
00:24:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.705153 23865 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } has no permanent_uuid. Determining permanent_uuid...
00:24:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.722883 23865 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } has no permanent_uuid. Determining permanent_uuid...
00:24:51.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.734710 23865 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } has no permanent_uuid. Determining permanent_uuid...
00:24:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.736158 23763 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.738005 23865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:51.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.787369 23865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 1
00:24:51.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.789510 23865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:51.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.857812 23865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 2
00:24:51.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.859922 23865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:51.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.915195 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } attempt: 5
00:24:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.922384 23723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } has no permanent_uuid. Determining permanent_uuid...
00:24:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.924669 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:51.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.951182 23865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 3
00:24:51.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.953598 23865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:51.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:51.975950 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 1
00:24:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:51.978281 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.046569 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 2
00:24:52.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.048899 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.072037 23870 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:24:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.072368 23870 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.072455 23870 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:24:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.080197 23870 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:24:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.080322 23870 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:24:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.080365 23870 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:24:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.080422 23870 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:24:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.080473 23870 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:24:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.080538 23870 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41133/jwks.json
00:24:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.090245 23870 master_runner.cc:386] Master server non-default flags:
00:24:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.188:46541
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41133/jwks.json
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data/info.pb
00:24:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.188
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/logs
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:24:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:24:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:24:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:24:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:24:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.091822 23870 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.094530 23870 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:24:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.096848 23865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 4
00:24:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.098924 23865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.109565 23893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.109542 23899 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.109510 23897 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.110420 23870 server_base.cc:1034] running on GCE node
00:24:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.111318 23870 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.113005 23870 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.114253 23870 hybrid_clock.cc:648] HybridClock initialized: now 1786494292114158 us; error 77 us; skew 500 ppm
00:24:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.114624 23870 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.117280 23870 webserver.cc:460] Webserver started at http://127.23.2.188:39987/ using document root <none> and password file <none>
00:24:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.118033 23870 fs_manager.cc:359] Metadata directory not provided
00:24:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.118147 23870 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.118438 23870 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:24:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.121464 23870 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data/instance:
00:24:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1d5997f1e4945abae4f3c7442c8139d"
00:24:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:52 on dist-test-slave-6bbx"
00:24:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.122339 23870 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal/instance:
00:24:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1d5997f1e4945abae4f3c7442c8139d"
00:24:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:52 on dist-test-slave-6bbx"
00:24:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.129822 23870 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.004s
00:24:52.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.134363 23909 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.136085 23870 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.136323 23870 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal
00:24:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1d5997f1e4945abae4f3c7442c8139d"
00:24:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:52 on dist-test-slave-6bbx"
00:24:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.136598 23870 fs_report.cc:383] FS layout report
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data/data
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:52.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.150904 23870 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:24:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.152009 23870 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.152367 23870 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:52.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.155138 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 3
00:24:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.157107 23723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.2.188:46541: Network error: Client connection negotiation failed: client connection to 127.23.2.188:46541: connect: Connection refused (error 111)
00:24:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.171731 23870 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.188:46541
00:24:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.171778 24007 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.188:46541 every 8 connection(s)
00:24:52.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.173399 23870 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/data/info.pb
00:24:52.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.176040 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 23870
00:24:52.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.176199 23562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/master-2/wal/instance
00:24:52.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.178527 24008 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:24:52.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.186022 24008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } has no permanent_uuid. Determining permanent_uuid...
00:24:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.201860 24008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } has no permanent_uuid. Determining permanent_uuid...
00:24:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.209139 24008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } has no permanent_uuid. Determining permanent_uuid...
00:24:52.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.220229 24008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: Bootstrap starting.
00:24:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.224429 24008 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: Neither blocks nor log segments found. Creating new log.
00:24:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.225685 24008 log.cc:824] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: Log is configured to *not* fsync() on all Append() calls
00:24:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.228641 24008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: No bootstrap required, opened a new log
00:24:52.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.237378 24008 raft_consensus.cc:348] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.237790 24008 raft_consensus.cc:374] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:24:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.237880 24008 raft_consensus.cc:729] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f1d5997f1e4945abae4f3c7442c8139d, State: Initialized, Role: FOLLOWER
00:24:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.238480 24008 consensus_queue.cc:260] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.239451 24018 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.239861 24018 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.240583 24008 sys_catalog.cc:564] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: configured and running, proceeding with master startup.
00:24:52.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.252795 24050 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:24:52.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.254340 24050 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:24:52.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.249533 23870 master_runner.cc:186] Error getting master registration for 127.23.2.190:44663: OK, instance_id { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" instance_seqno: 1786494291209013 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:24:52.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.259181 23870 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:24:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.334375 23723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 4
00:24:52.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.346066 23723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: Bootstrap starting.
00:24:52.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.349552 23723 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: Neither blocks nor log segments found. Creating new log.
00:24:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.350785 23723 log.cc:824] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: Log is configured to *not* fsync() on all Append() calls
00:24:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.354035 23723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: No bootstrap required, opened a new log
00:24:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.362001 23723 raft_consensus.cc:348] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.362416 23723 raft_consensus.cc:374] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:24:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.362545 23723 raft_consensus.cc:729] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9e0ab45df254654bb01e486a8f5765f, State: Initialized, Role: FOLLOWER
00:24:52.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.363304 23723 consensus_queue.cc:260] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.364373 24056 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.364876 24056 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.365610 23723 sys_catalog.cc:564] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: configured and running, proceeding with master startup.
00:24:52.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.383559 24071 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:24:52.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.384200 24071 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:24:52.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.375188 23586 master_runner.cc:186] Error getting master registration for 127.23.2.189:41391: OK, instance_id { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" instance_seqno: 1786494291684246 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:24:52.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.384934 23586 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:24:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.398178 23865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } attempt: 5
00:24:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.404965 23920 tablet.cc:2369] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:24:52.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.414079 23865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: Bootstrap starting.
00:24:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.418677 23865 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: Neither blocks nor log segments found. Creating new log.
00:24:52.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.420301 23865 log.cc:824] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: Log is configured to *not* fsync() on all Append() calls
00:24:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.428572 23865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: No bootstrap required, opened a new log
00:24:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.440647 23865 raft_consensus.cc:348] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.441258 23865 raft_consensus.cc:374] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:24:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.441404 23865 raft_consensus.cc:729] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 432d1e1bcdf04aa39fd8d4ff23b05f4f, State: Initialized, Role: FOLLOWER
00:24:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.442353 23865 consensus_queue.cc:260] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.447299 24077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.448175 24077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.448861 23621 tablet.cc:2369] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:24:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.448993 23865 sys_catalog.cc:564] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: configured and running, proceeding with master startup.
00:24:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.472615 23728 master_runner.cc:263] No leader master found from master 432d1e1bcdf04aa39fd8d4ff23b05f4f
00:24:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.472831 23728 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:24:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.476816 24091 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:24:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:52.478525 24091 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:24:52.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.510979 24018 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:52.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.511343 24018 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.513896 24018 leader_election.cc:288] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e9e0ab45df254654bb01e486a8f5765f (127.23.2.190:44663), 432d1e1bcdf04aa39fd8d4ff23b05f4f (127.23.2.189:41391)
00:24:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.514405 23676 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f1d5997f1e4945abae4f3c7442c8139d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9e0ab45df254654bb01e486a8f5765f" is_pre_election: true
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.515100 23676 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f1d5997f1e4945abae4f3c7442c8139d in term 0.
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.515872 23913 leader_election.cc:302] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e9e0ab45df254654bb01e486a8f5765f, f1d5997f1e4945abae4f3c7442c8139d; no voters: 
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.516409 24018 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Leader pre-election won for term 1
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.516568 24018 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.516670 24018 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 0 FOLLOWER]: Advancing to term 1
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.517347 23823 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f1d5997f1e4945abae4f3c7442c8139d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" is_pre_election: true
00:24:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.517988 23823 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f1d5997f1e4945abae4f3c7442c8139d in term 0.
00:24:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.522505 24018 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.527456 23676 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f1d5997f1e4945abae4f3c7442c8139d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9e0ab45df254654bb01e486a8f5765f"
00:24:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.527792 23676 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 0 FOLLOWER]: Advancing to term 1
00:24:52.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.532186 23676 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f1d5997f1e4945abae4f3c7442c8139d in term 1.
00:24:52.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.532828 23913 leader_election.cc:302] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e9e0ab45df254654bb01e486a8f5765f, f1d5997f1e4945abae4f3c7442c8139d; no voters: 
00:24:52.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.535761 23823 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f1d5997f1e4945abae4f3c7442c8139d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f"
00:24:52.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.535876 24018 leader_election.cc:288] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [CANDIDATE]: Term 1 election: Requested vote from peers e9e0ab45df254654bb01e486a8f5765f (127.23.2.190:44663), 432d1e1bcdf04aa39fd8d4ff23b05f4f (127.23.2.189:41391)
00:24:52.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.536082 23823 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 0 FOLLOWER]: Advancing to term 1
00:24:52.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.536164 24018 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 1 FOLLOWER]: Leader election won for term 1
00:24:52.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.536321 24018 raft_consensus.cc:686] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [term 1 LEADER]: Becoming Leader. State: Replica: f1d5997f1e4945abae4f3c7442c8139d, State: Running, Role: LEADER
00:24:52.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.536993 24018 consensus_queue.cc:237] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } }
00:24:52.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.540478 23823 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f1d5997f1e4945abae4f3c7442c8139d in term 1.
00:24:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.546051 24018 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f1d5997f1e4945abae4f3c7442c8139d. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.547909 24018 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: This master's current role is: LEADER
00:24:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.564363 24097 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
00:24:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.567157 24097 catalog_manager.cc:1484] Initializing Kudu cluster ID...
00:24:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.589399 23823 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [term 1 FOLLOWER]: Refusing update from remote peer f1d5997f1e4945abae4f3c7442c8139d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:24:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.589406 23676 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [term 1 FOLLOWER]: Refusing update from remote peer f1d5997f1e4945abae4f3c7442c8139d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:24:52.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.590636 24095 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:24:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.591718 24018 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [LEADER]: Connected to new peer: Peer: permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:24:52.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.616978 24077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f1d5997f1e4945abae4f3c7442c8139d. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.617364 24077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.622532 24018 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.622875 24018 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: This master's current role is: LEADER
00:24:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.627280 24100 mvcc.cc:205] Tried to move back new op lower bound from 7317480622432821248 to 7317480622236864512. Current Snapshot: MvccSnapshot[applied={T|T < 7317480622432821248}]
00:24:52.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.636206 24077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.636554 24077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.642764 24097 catalog_manager.cc:1347] Generated new cluster ID: bdd7e57115b640ddb1d7679c90bb2b5c
00:24:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.642905 24097 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
00:24:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.659420 24056 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f1d5997f1e4945abae4f3c7442c8139d. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.659938 24056 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.668401 24095 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.669046 24095 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d [sys.catalog]: This master's current role is: LEADER
00:24:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.671540 24099 mvcc.cc:205] Tried to move back new op lower bound from 7317480622432821248 to 7317480622236864512. Current Snapshot: MvccSnapshot[applied={T|T < 7317480622432821248}]
00:24:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.688150 24056 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f1d5997f1e4945abae4f3c7442c8139d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9e0ab45df254654bb01e486a8f5765f" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 44663 } } peers { permanent_uuid: "432d1e1bcdf04aa39fd8d4ff23b05f4f" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 41391 } } peers { permanent_uuid: "f1d5997f1e4945abae4f3c7442c8139d" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 46541 } } }
00:24:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.688477 24056 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f [sys.catalog]: This master's current role is: FOLLOWER
00:24:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.706633 24097 catalog_manager.cc:1370] Generated new certificate authority record
00:24:52.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.708904 24097 catalog_manager.cc:1504] Loading token signing keys...
00:24:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.739965 24097 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f1d5997f1e4945abae4f3c7442c8139d: Generated new TSK 0
00:24:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.742252 24097 catalog_manager.cc:1514] Initializing in-progress tserver states...
00:24:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:52.762490 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:24:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:0
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.184855 24111 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:24:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.185184 24111 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.185279 24111 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:24:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.193311 24111 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:24:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.193537 24111 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.129
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.203787 24111 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:0
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:24:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:24:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:24:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:24:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.205317 24111 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.208228 24111 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:24:53.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.227569 24128 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:53.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.227586 24127 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:53.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.228634 24130 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.228715 24111 server_base.cc:1034] running on GCE node
00:24:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.229645 24111 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:53.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.230829 24111 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.232259 24111 hybrid_clock.cc:648] HybridClock initialized: now 1786494293232176 us; error 49 us; skew 500 ppm
00:24:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.232635 24111 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:53.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.235059 24111 webserver.cc:460] Webserver started at http://127.23.2.129:35549/ using document root <none> and password file <none>
00:24:53.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.235905 24111 fs_manager.cc:359] Metadata directory not provided
00:24:53.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.236052 24111 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:53.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.236415 24111 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:24:53.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.239311 24111 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/instance:
00:24:53.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5635f94622f74554984e41143786b602"
00:24:53.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:24:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.240274 24111 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal/instance:
00:24:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5635f94622f74554984e41143786b602"
00:24:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:24:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.246459 24111 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
00:24:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.250597 24141 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:53.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.252319 24111 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:53.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.252535 24111 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:24:53.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5635f94622f74554984e41143786b602"
00:24:53.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.252794 24111 fs_report.cc:383] FS layout report
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/data
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.265071 24111 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:24:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.266072 24111 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.266479 24111 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.267653 24111 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:24:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.269580 24111 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:24:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.269675 24111 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:24:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.269810 24111 ts_tablet_manager.cc:610] Registered 0 tablets
00:24:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.269876 24111 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:24:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.303426 24111 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.129:44273
00:24:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.303473 24356 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.129:44273 every 8 connection(s)
00:24:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.304953 24111 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:24:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.312115 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 24111
00:24:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.312373 23562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal/instance
00:24:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.320510 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:0
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:53.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.333277 24357 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:24:53.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.333676 24357 heartbeater.cc:464] Registering TS with master...
00:24:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.334637 24357 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:24:53.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.335482 24358 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:24:53.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.335628 24358 heartbeater.cc:464] Registering TS with master...
00:24:53.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.336103 24358 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:24:53.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.338084 23937 ts_manager.cc:194] Registered new tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:24:53.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.338176 23653 ts_manager.cc:194] Registered new tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:24:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.339080 24360 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:24:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.339201 24360 heartbeater.cc:464] Registering TS with master...
00:24:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.339632 24360 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:24:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.340170 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.129:59939
00:24:53.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.341524 23806 ts_manager.cc:194] Registered new tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:24:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.387085 24071 catalog_manager.cc:1259] Loaded cluster ID: bdd7e57115b640ddb1d7679c90bb2b5c
00:24:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.387219 24071 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: loading cluster ID for follower catalog manager: success
00:24:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.390872 24071 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: acquiring CA information for follower catalog manager: success
00:24:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.393496 24071 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e9e0ab45df254654bb01e486a8f5765f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:24:53.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.481635 24091 catalog_manager.cc:1259] Loaded cluster ID: bdd7e57115b640ddb1d7679c90bb2b5c
00:24:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.481760 24091 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: loading cluster ID for follower catalog manager: success
00:24:53.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.484680 24091 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: acquiring CA information for follower catalog manager: success
00:24:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.486795 24091 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 432d1e1bcdf04aa39fd8d4ff23b05f4f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:24:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.700785 24380 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:24:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.701246 24380 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.701386 24380 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:24:53.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.712971 24380 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:24:53.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.713164 24380 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.130
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.722711 24380 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:0
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:24:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:24:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:24:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:24:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.724277 24380 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.727015 24380 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:24:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.743860 24389 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.743784 24390 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:53.743824 24393 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:53.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.745005 24380 server_base.cc:1034] running on GCE node
00:24:53.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.745741 24380 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.746884 24380 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.748131 24380 hybrid_clock.cc:648] HybridClock initialized: now 1786494293748035 us; error 64 us; skew 500 ppm
00:24:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.748454 24380 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.754129 24380 webserver.cc:460] Webserver started at http://127.23.2.130:44417/ using document root <none> and password file <none>
00:24:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.755165 24380 fs_manager.cc:359] Metadata directory not provided
00:24:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.755295 24380 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.755590 24380 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:24:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.759090 24380 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/instance:
00:24:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:24:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:24:53.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.760015 24380 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal/instance:
00:24:53.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:24:53.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:24:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.766014 24380 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.001s	sys 0.004s
00:24:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.770083 24404 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:53.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.771759 24380 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.771926 24380 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.772117 24380 fs_report.cc:383] FS layout report
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/data
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.786576 24380 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:24:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.787465 24380 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.787832 24380 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.788929 24380 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:24:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.790656 24380 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:24:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.790741 24380 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:24:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.790871 24380 ts_tablet_manager.cc:610] Registered 0 tablets
00:24:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.790927 24380 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:24:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.825630 24380 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.130:42999
00:24:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.825681 24601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.130:42999 every 8 connection(s)
00:24:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.827047 24380 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:24:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.832170 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 24380
00:24:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.832338 23562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal/instance
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.838946 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:0
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.858892 24602 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.858950 24606 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.859313 24602 heartbeater.cc:464] Registering TS with master...
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.859391 24606 heartbeater.cc:464] Registering TS with master...
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.859397 24605 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.859555 24605 heartbeater.cc:464] Registering TS with master...
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.860610 24605 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:24:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.860668 24602 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:24:53.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.861522 24606 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:24:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.862535 23937 ts_manager.cc:194] Registered new tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.862664 23653 ts_manager.cc:194] Registered new tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.863059 23806 ts_manager.cc:194] Registered new tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:53.863770 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.130:52063
00:24:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.207438 24619 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:24:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.207821 24619 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:24:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.207955 24619 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:24:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.215613 24619 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:24:54.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.215883 24619 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.131
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.225857 24619 tablet_server_runner.cc:78] Tablet server non-default flags:
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:0
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:24:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:24:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.227766 24619 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:24:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.229710 24619 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:24:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.244256 24633 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.244256 24630 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:54.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:54.244664 24629 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:24:54.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.245687 24619 server_base.cc:1034] running on GCE node
00:24:54.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.246397 24619 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:24:54.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.247467 24619 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:24:54.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.248709 24619 hybrid_clock.cc:648] HybridClock initialized: now 1786494294248628 us; error 91 us; skew 500 ppm
00:24:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.249073 24619 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:24:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.251627 24619 webserver.cc:460] Webserver started at http://127.23.2.131:46861/ using document root <none> and password file <none>
00:24:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.252444 24619 fs_manager.cc:359] Metadata directory not provided
00:24:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.252558 24619 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:24:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.252867 24619 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:24:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.256122 24619 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/instance:
00:24:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:24:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:24:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.257020 24619 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal/instance:
00:24:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:24:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:24:54.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.262764 24619 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.000s
00:24:54.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.266891 24650 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:24:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.268474 24619 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
00:24:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.268635 24619 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:24:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:24:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.268818 24619 fs_report.cc:383] FS layout report
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/data
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:24:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:24:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.286177 24619 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:24:54.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.287046 24619 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:24:54.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.287345 24619 kserver.cc:163] Server-wide thread pool size limit: 3276
00:24:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.288398 24619 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:24:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.290027 24619 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:24:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.290107 24619 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:24:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.290207 24619 ts_tablet_manager.cc:610] Registered 0 tablets
00:24:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.290242 24619 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:24:54.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.323490 24619 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.131:35751
00:24:54.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.323542 24879 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.131:35751 every 8 connection(s)
00:24:54.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.324875 24619 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:24:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.325759 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 24619
00:24:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.325961 23562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal/instance
00:24:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.344010 24357 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:24:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.353088 24882 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:24:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.353482 24882 heartbeater.cc:464] Registering TS with master...
00:24:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.354390 24882 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:24:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.355209 24884 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:24:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.355374 24884 heartbeater.cc:464] Registering TS with master...
00:24:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.355839 24884 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:24:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.356120 24881 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:24:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.356256 24881 heartbeater.cc:464] Registering TS with master...
00:24:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.356259 23652 ts_manager.cc:194] Registered new tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:24:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.356637 24881 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:24:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.356981 23806 ts_manager.cc:194] Registered new tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:24:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.357513 23937 ts_manager.cc:194] Registered new tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:24:54.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.358546 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.131:49519
00:24:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.370332 23562 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
00:24:54.372 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:24:54.373 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

00:24:54.384 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.23.2.190"
      port: 44663
    }
    bound_http_address {
      host: "127.23.2.190"
      port: 42115
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.23.2.189"
      port: 41391
    }
    bound_http_address {
      host: "127.23.2.189"
      port: 32921
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.23.2.188"
      port: 46541
    }
    bound_http_address {
      host: "127.23.2.188"
      port: 39987
    }
  }
}

00:24:54.393 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

00:24:54.396 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.23.2.129"
      port: 44273
    }
    bound_http_address {
      host: "127.23.2.129"
      port: 35549
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.23.2.130"
      port: 42999
    }
    bound_http_address {
      host: "127.23.2.130"
      port: 44417
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.23.2.131"
      port: 35751
    }
    bound_http_address {
      host: "127.23.2.131"
      port: 46861
    }
  }
}

00:24:54.396 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:24:54.584 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
00:24:54.774 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.2.190:44663(127.23.2.190:44663)] connecting to peer
00:24:54.827 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.2.188:46541(127.23.2.188:46541)] connecting to peer
00:24:54.833 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.2.189:41391(127.23.2.189:41391)] connecting to peer
00:24:54.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:54.866966 24602 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:24:54.869 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.2.188:46541(127.23.2.188:46541)] Successfully connected to peer
00:24:54.869 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.2.190:44663(127.23.2.190:44663)] Successfully connected to peer
00:24:54.954 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.2.189:41391(127.23.2.189:41391)] Successfully connected to peer
00:24:54.979 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.2.188
00:24:54.979 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.2.190
00:24:55.093 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.2.189
00:24:55.227 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb618f5d9, L:/127.0.0.1:48404 - R:/127.23.2.190:44663] using SASL/PLAIN
00:24:55.227 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xfc11788e, L:/127.0.0.1:43892 - R:/127.23.2.188:46541] using SASL/PLAIN
00:24:55.231 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x78489c60, L:/127.0.0.1:43396 - R:/127.23.2.189:41391] using SASL/PLAIN
00:24:55.271 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
00:24:55.272 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=539), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.2.190:44663, [90ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.2.188:46541, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.2.189:41391, [503ms] Sub RPC ConnectToMaster: received response from server master-127.23.2.188:46541: OK, [503ms] Sub RPC ConnectToMaster: received response from server master-127.23.2.190:44663: OK, [506ms] Sub RPC ConnectToMaster: received response from server master-127.23.2.189:41391: OK, deferred=null) after lookup
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.335717 23937 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43892:
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721"
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:24:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:24:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:55.337553 23937 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
00:24:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.361130 24881 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:24:55.379 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 2e30f494a8db496abae4db6c063e9d9c
00:24:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.387965 24236 tablet_service.cc:1467] Processing CreateTablet for tablet e315584218244a9d9aabe564d716375a (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721 [id=2e30f494a8db496abae4db6c063e9d9c]), partition=RANGE (key) PARTITION UNBOUNDED
00:24:55.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.389501 24236 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e315584218244a9d9aabe564d716375a. 1 dirs total, 0 dirs full, 0 dirs failed
00:24:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.391906 24757 tablet_service.cc:1467] Processing CreateTablet for tablet e315584218244a9d9aabe564d716375a (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721 [id=2e30f494a8db496abae4db6c063e9d9c]), partition=RANGE (key) PARTITION UNBOUNDED
00:24:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.393200 24757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e315584218244a9d9aabe564d716375a. 1 dirs total, 0 dirs full, 0 dirs failed
00:24:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.394585 24485 tablet_service.cc:1467] Processing CreateTablet for tablet e315584218244a9d9aabe564d716375a (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721 [id=2e30f494a8db496abae4db6c063e9d9c]), partition=RANGE (key) PARTITION UNBOUNDED
00:24:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.396292 24485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e315584218244a9d9aabe564d716375a. 1 dirs total, 0 dirs full, 0 dirs failed
00:24:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.409659 24955 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap starting.
00:24:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.412808 24956 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap starting.
00:24:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.414516 24955 tablet_bootstrap.cc:654] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Neither blocks nor log segments found. Creating new log.
00:24:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.414649 24958 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap starting.
00:24:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.416929 24955 log.cc:824] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Log is configured to *not* fsync() on all Append() calls
00:24:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.418159 24958 tablet_bootstrap.cc:654] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Neither blocks nor log segments found. Creating new log.
00:24:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.418795 24956 tablet_bootstrap.cc:654] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Neither blocks nor log segments found. Creating new log.
00:24:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.419488 24958 log.cc:824] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Log is configured to *not* fsync() on all Append() calls
00:24:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.420074 24956 log.cc:824] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Log is configured to *not* fsync() on all Append() calls
00:24:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.426126 24956 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: No bootstrap required, opened a new log
00:24:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.426196 24955 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: No bootstrap required, opened a new log
00:24:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.426196 24958 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: No bootstrap required, opened a new log
00:24:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.426479 24955 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent bootstrapping tablet: real 0.017s	user 0.010s	sys 0.004s
00:24:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.426476 24956 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent bootstrapping tablet: real 0.014s	user 0.004s	sys 0.005s
00:24:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.426481 24958 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent bootstrapping tablet: real 0.013s	user 0.010s	sys 0.000s
00:24:55.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.433996 24956 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.434855 24956 raft_consensus.cc:374] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:24:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.435165 24956 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Initialized, Role: FOLLOWER
00:24:55.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.436359 24956 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.437234 24958 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.437932 24958 raft_consensus.cc:374] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:24:55.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.438189 24958 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Initialized, Role: FOLLOWER
00:24:55.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.438223 24955 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.438676 24955 raft_consensus.cc:374] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:24:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.438922 24955 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5635f94622f74554984e41143786b602, State: Initialized, Role: FOLLOWER
00:24:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.439455 24958 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.439867 24955 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.442004 24956 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent starting tablet: real 0.015s	user 0.001s	sys 0.013s
00:24:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.448925 24958 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent starting tablet: real 0.022s	user 0.012s	sys 0.007s
00:24:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.450867 24955 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent starting tablet: real 0.024s	user 0.004s	sys 0.010s
00:24:55.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.485225 24967 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:55.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.485584 24967 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.487457 24967 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.496654 24794 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:24:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.496670 24517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:24:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.497189 24794 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 0.
00:24:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.497205 24517 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 0.
00:24:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.497854 24145 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: 
00:24:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.498250 24967 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:24:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.498427 24967 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:24:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.498539 24967 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 0 FOLLOWER]: Advancing to term 1
00:24:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.500720 24967 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.501380 24967 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 1 election: Requested vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.501947 24794 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052"
00:24:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.501978 24517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:24:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.502189 24794 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 0 FOLLOWER]: Advancing to term 1
00:24:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.502203 24517 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 0 FOLLOWER]: Advancing to term 1
00:24:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.504379 24794 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 1.
00:24:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.504443 24517 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 1.
00:24:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.504855 24145 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: 
00:24:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.505201 24967 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 FOLLOWER]: Leader election won for term 1
00:24:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.505656 24967 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 LEADER]: Becoming Leader. State: Replica: 5635f94622f74554984e41143786b602, State: Running, Role: LEADER
00:24:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.506208 24967 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:55.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.511431 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5635f94622f74554984e41143786b602 (127.23.2.129). New cstate: current_term: 1 leader_uuid: "5635f94622f74554984e41143786b602" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:24:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:55.560053 24361 tablet.cc:2369] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:24:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:55.582062 24608 tablet.cc:2369] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:24:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:55.588320 24885 tablet.cc:2369] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:24:55.600 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 2e30f494a8db496abae4db6c063e9d9c
00:24:55.620 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:55.627 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:55.629 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=null) after lookup
00:24:55.639 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:55.640 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] Successfully connected to peer
00:24:55.693 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8dd12d47, L:/127.0.0.1:35780 - R:/127.23.2.129:44273] using TOKEN/null
00:24:55.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.704469 24517 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Refusing update from remote peer 5635f94622f74554984e41143786b602: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:24:55.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.704469 24794 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Refusing update from remote peer 5635f94622f74554984e41143786b602: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:24:55.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.705679 24970 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:24:55.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.706053 24967 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Connected to new peer: Peer: permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:24:55.814 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:24:55.816 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.129:44273
00:24:55.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:24:55.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.819077 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 24111
00:24:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.829602 24351 generic_service.cc:196] Checking for leaks (request via RPC)
00:24:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:55.937453 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 24111
00:24:55.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:24:55.971 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] handling channelInactive
00:24:55.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state READY due to: connection closed
00:24:55.973 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: connection closed
00:24:55.998 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:55.998 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:55.999 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test)) after lookup
00:24:56.000 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:56.002 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.002 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.002 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.018 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:56.018 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:56.019 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575))) after lookup
00:24:56.020 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:56.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.022 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.059 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:56.059 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:56.060 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=90), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [51ms] sending RPC to server 5635f94622f74554984e41143786b602, [53ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [88ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770))) after lookup
00:24:56.061 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:56.063 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.063 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.063 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.137 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:56.138 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:56.138 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=169), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [51ms] sending RPC to server 5635f94622f74554984e41143786b602, [53ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [88ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [92ms] sending RPC to server 5635f94622f74554984e41143786b602, [94ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [168ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474))) after lookup
00:24:56.139 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:56.140 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.140 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.141 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.398 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:56.398 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:56.403 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=433), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [51ms] sending RPC to server 5635f94622f74554984e41143786b602, [53ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [88ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [92ms] sending RPC to server 5635f94622f74554984e41143786b602, [94ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [168ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [170ms] sending RPC to server 5635f94622f74554984e41143786b602, [172ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [172ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [425ms] refreshing cache from master, [425ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [428ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161))) after lookup
00:24:56.404 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:56.406 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.406 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.877 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:56.877 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:56.878 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=909), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [51ms] sending RPC to server 5635f94622f74554984e41143786b602, [53ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [88ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [92ms] sending RPC to server 5635f94622f74554984e41143786b602, [94ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [168ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [170ms] sending RPC to server 5635f94622f74554984e41143786b602, [172ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [172ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [425ms] refreshing cache from master, [425ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [428ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [434ms] sending RPC to server 5635f94622f74554984e41143786b602, [437ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [437ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [904ms] refreshing cache from master, [904ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [907ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161) -> (continuation of Deferred@584500394 after retry RPC@19167834), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161) -> (continuation of Deferred@584500394 after retry RPC@19167834))) after lookup
00:24:56.879 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:56.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:56.881 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:57.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.265097 24993 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 5635f94622f74554984e41143786b602)
00:24:57.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.265352 24994 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 5635f94622f74554984e41143786b602)
00:24:57.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.265524 24994 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:57.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.265709 24993 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:57.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.267361 24994 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:24:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:57.268622 24407 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.268683 24993 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:57.268940 24656 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:57.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:57.271926 24407 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:57.273075 24656 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.277127 24794 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:24:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.277422 24794 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 1.
00:24:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.277945 24408 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:24:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.278357 24517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:24:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.278488 24994 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Leader pre-election won for term 2
00:24:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.278646 24517 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 1.
00:24:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.278820 24994 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 5635f94622f74554984e41143786b602)
00:24:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.278923 24994 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 1 FOLLOWER]: Advancing to term 2
00:24:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.279184 24658 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:24:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.279624 24993 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Leader pre-election won for term 2
00:24:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.279778 24993 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 5635f94622f74554984e41143786b602)
00:24:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.279875 24993 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 1 FOLLOWER]: Advancing to term 2
00:24:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.281796 24994 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.282600 24994 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 2 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:24:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.282747 24993 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.283221 24794 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052"
00:24:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.283532 24993 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 2 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.283608 24794 raft_consensus.cc:2379] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate a24b7a1608504a7eb4e2421899b73f63 in current term 2: Already voted for candidate 24637a1459374c63935f5fc8cd1b1052 in this term.
00:24:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.284150 24517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:24:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.284475 24517 raft_consensus.cc:2379] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 24637a1459374c63935f5fc8cd1b1052 in current term 2: Already voted for candidate a24b7a1608504a7eb4e2421899b73f63 in this term.
00:24:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:57.285162 24407 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.285279 24407 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602
00:24:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.285526 24994 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:24:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:57.285907 24656 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.286085 24656 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052; no voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63
00:24:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:57.286429 24993 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:24:57.878 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:24:57.878 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:24:57.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1909), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [51ms] sending RPC to server 5635f94622f74554984e41143786b602, [53ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [88ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [92ms] sending RPC to server 5635f94622f74554984e41143786b602, [94ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [168ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [170ms] sending RPC to server 5635f94622f74554984e41143786b602, [172ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [172ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [425ms] refreshing cache from master, [425ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [428ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [434ms] sending RPC to server 5635f94622f74554984e41143786b602, [437ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [437ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [904ms] refreshing cache from master, [904ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [907ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [910ms] sending RPC to server 5635f94622f74554984e41143786b602, [911ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [912ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [1905ms] refreshing cache from master, [1905ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [1908ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161) -> (continuation of Deferred@584500394 after retry RPC@19167834) -> (continuation of Deferred@2012316709 after retry RPC@601353929), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161) -> (continuation of Deferred@584500394 after retry RPC@19167834) -> (continuation of Deferred@2012316709 after retry RPC@601353929))) after lookup
00:24:57.881 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:24:57.883 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:57.883 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:57.883 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:24:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.047775 25007 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:24:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.047964 25007 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.048770 25007 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.049497 24517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:24:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.049777 24517 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 2.
00:24:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.050338 24658 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 
00:24:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.050613 25007 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Leader pre-election won for term 3
00:24:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.050765 25007 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:24:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.051028 25007 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 2 FOLLOWER]: Advancing to term 3
00:24:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:59.051226 24656 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:59.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.053375 25007 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.054073 25007 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 3 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:24:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.054709 24517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:24:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.054939 24517 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 2 FOLLOWER]: Advancing to term 3
00:24:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:59.055799 24656 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:24:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.057205 24517 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 3.
00:24:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.057659 24658 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:24:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.057940 25007 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 3 FOLLOWER]: Leader election won for term 3
00:24:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.058336 25007 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 3 LEADER]: Becoming Leader. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Running, Role: LEADER
00:24:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.058845 25007 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:24:59.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.062656 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 reported cstate change: term changed from 1 to 3, leader changed from 5635f94622f74554984e41143786b602 (127.23.2.129) to 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131). New cstate: current_term: 3 leader_uuid: "24637a1459374c63935f5fc8cd1b1052" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:24:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:24:59.506062 24656 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 -> Peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Couldn't send request to peer 5635f94622f74554984e41143786b602. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:24:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.650843 24517 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Refusing update from remote peer 24637a1459374c63935f5fc8cd1b1052: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
00:24:59.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:24:59.651804 25007 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
00:25:00.338 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:00.338 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:00.340 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=4369), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [3ms] received response from server 5635f94622f74554984e41143786b602: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [28ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [30ms] sending RPC to server 5635f94622f74554984e41143786b602, [33ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [51ms] sending RPC to server 5635f94622f74554984e41143786b602, [53ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [88ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [92ms] sending RPC to server 5635f94622f74554984e41143786b602, [94ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [168ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [170ms] sending RPC to server 5635f94622f74554984e41143786b602, [172ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [172ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [425ms] refreshing cache from master, [425ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [428ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [434ms] sending RPC to server 5635f94622f74554984e41143786b602, [437ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [437ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [904ms] refreshing cache from master, [904ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [907ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [910ms] sending RPC to server 5635f94622f74554984e41143786b602, [911ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [912ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [1905ms] refreshing cache from master, [1905ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [1908ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [1911ms] sending RPC to server 5635f94622f74554984e41143786b602, [1914ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [1915ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [4365ms] refreshing cache from master, [4365ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [4368ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1512972028(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@221e57b0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161) -> (continuation of Deferred@584500394 after retry RPC@19167834) -> (continuation of Deferred@2012316709 after retry RPC@601353929) -> (continuation of Deferred@1478105590 after retry RPC@1151796634), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fc4387a -> wakeup thread Time-limited test -> (continuation of Deferred@1563404848 after retry RPC@66477575) -> (continuation of Deferred@2011027554 after retry RPC@169477770) -> (continuation of Deferred@431225784 after retry RPC@1128763474) -> (continuation of Deferred@1420064637 after retry RPC@2085348161) -> (continuation of Deferred@584500394 after retry RPC@19167834) -> (continuation of Deferred@2012316709 after retry RPC@601353929) -> (continuation of Deferred@1478105590 after retry RPC@1151796634))) after lookup
00:25:00.341 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:00.342 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] Successfully connected to peer
00:25:00.370 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xccca33c1, L:/127.0.0.1:48052 - R:/127.23.2.131:35751] using TOKEN/null
00:25:00.394 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.129:44273
00:25:00.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.397469 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:44273
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35549
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:00.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.751264 25061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:00.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.751642 25061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.751793 25061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:00.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.759730 25061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:00.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.759970 25061 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.129
00:25:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.770514 25061 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:44273
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35549
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.772487 25061 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:00.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.774749 25061 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:00.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.790571 25074 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:00.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.790612 25072 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:00.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:00.790623 25070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:00.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.791360 25061 server_base.cc:1034] running on GCE node
00:25:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.792140 25061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:00.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.793215 25061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:00.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.794478 25061 hybrid_clock.cc:648] HybridClock initialized: now 1786494300794387 us; error 57 us; skew 500 ppm
00:25:00.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.794866 25061 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:00.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.797784 25061 webserver.cc:460] Webserver started at http://127.23.2.129:35549/ using document root <none> and password file <none>
00:25:00.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.798655 25061 fs_manager.cc:359] Metadata directory not provided
00:25:00.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.798811 25061 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:00.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.805225 25061 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
00:25:00.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.808823 25087 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:00.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.810456 25061 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
00:25:00.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.810669 25061 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5635f94622f74554984e41143786b602"
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.811941 25061 fs_report.cc:383] FS layout report
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/data
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.824443 25061 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:00.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.825527 25061 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.825889 25061 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.827137 25061 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.829607 25098 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.836099 25061 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.836249 25061 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.836385 25061 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.839956 25061 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.840086 25061 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.004s	sys 0.000s
00:25:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.840544 25098 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap starting.
00:25:00.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.870107 25098 log.cc:824] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Log is configured to *not* fsync() on all Append() calls
00:25:00.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.889576 25061 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.129:44273
00:25:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.889580 25310 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.129:44273 every 8 connection(s)
00:25:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.890902 25098 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.891428 25061 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.891598 25098 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap complete.
00:25:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.893373 25098 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent bootstrapping tablet: real 0.053s	user 0.038s	sys 0.013s
00:25:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.899014 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 25061
00:25:00.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.911640 25098 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.913517 25098 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5635f94622f74554984e41143786b602, State: Initialized, Role: FOLLOWER
00:25:00.923 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:00.926 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.131:35751
00:25:00.926 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:00.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.926885 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 24619
00:25:00.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.930266 25098 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.935467 25098 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent starting tablet: real 0.042s	user 0.014s	sys 0.001s
00:25:00.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.949656 25312 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.950168 25312 heartbeater.cc:464] Registering TS with master...
00:25:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.951542 25312 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:00.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.955951 23652 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.958791 25319 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.958966 25319 heartbeater.cc:464] Registering TS with master...
00:25:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.959455 25319 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.961185 23802 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:00.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.964641 25311 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:00.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.964798 25311 heartbeater.cc:464] Registering TS with master...
00:25:00.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.965245 25311 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:00.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.966883 23937 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.968096 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.129:53969
00:25:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.968216 24871 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:00.971019 25311 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:01.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:01.066789 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 24619
00:25:01.093 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] handling channelInactive
00:25:01.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state READY due to: connection closed
00:25:01.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:01.094 [DEBUG - Time-limited test] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:01.094786 25340 negotiation.cc:337] Failed RPC negotiation. Trace:
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.066321 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.23.2.131:33279 (local address 127.23.2.129:44273)
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.070444 (+  4123us) server_negotiation.cc:206] Beginning negotiation
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.070455 (+    11us) server_negotiation.cc:399] Waiting for connection header
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.070500 (+    45us) server_negotiation.cc:407] Connection header received
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.070628 (+   128us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.070637 (+     9us) server_negotiation.cc:461] Received NEGOTIATE request from client
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.070982 (+   345us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
00:25:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.071261 (+   279us) server_negotiation.cc:226] Negotiated authn=CERTIFICATE
00:25:01.096 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:01.093233 (+ 21972us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.23.2.131:33279: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:25:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3898,"thread_start_us":409,"threads_started":1}
00:25:01.096 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.096 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.118 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:01.118 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:01.119 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@741328879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@348eb77b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8664c0e -> wakeup thread Time-limited test)) after lookup
00:25:01.119 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:01.120 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.121 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.138 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:01.138 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:01.139 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [27ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [27ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [43ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@741328879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@348eb77b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8664c0e -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761))) after lookup
00:25:01.140 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:01.142 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.142 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.142 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.177 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:01.178 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:01.179 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [27ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [27ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [43ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [48ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [48ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [82ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@741328879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@348eb77b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8664c0e -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773))) after lookup
00:25:01.181 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:01.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.182 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.237 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:01.238 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:01.238 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=144), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [27ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [27ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [43ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [48ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [48ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [82ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [86ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [88ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [89ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [140ms] refreshing cache from master, [140ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [143ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@741328879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@348eb77b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773) -> (continuation of Deferred@1698667711 after retry RPC@1314430699), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8664c0e -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773) -> (continuation of Deferred@1698667711 after retry RPC@1314430699))) after lookup
00:25:01.239 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:01.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.241 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.657 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:01.658 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:01.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=565), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [27ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [27ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [43ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [48ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [48ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [82ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [86ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [88ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [89ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [140ms] refreshing cache from master, [140ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [143ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [145ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [147ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [147ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [559ms] refreshing cache from master, [559ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [562ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@741328879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@348eb77b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773) -> (continuation of Deferred@1698667711 after retry RPC@1314430699) -> (continuation of Deferred@153907285 after retry RPC@50923613), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8664c0e -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773) -> (continuation of Deferred@1698667711 after retry RPC@1314430699) -> (continuation of Deferred@153907285 after retry RPC@50923613))) after lookup
00:25:01.660 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:01.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:01.662 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.436619 25342 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.436864 25342 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.437597 25342 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:02.439409 24408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111) [suppressed 1 similar messages]
00:25:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:02.441110 24408 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.446452 25219 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602" is_pre_election: true
00:25:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.446955 25219 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 1.
00:25:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.447962 24407 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.448275 25342 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Leader pre-election won for term 4
00:25:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.448395 25342 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.448469 25342 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 3 FOLLOWER]: Advancing to term 4
00:25:02.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.450459 25342 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:02.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.451054 25342 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 4 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:02.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.451560 25219 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602"
00:25:02.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.451787 25219 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 1 FOLLOWER]: Advancing to term 4
00:25:02.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:02.452636 24408 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.455928 25219 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 4.
00:25:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.456367 24407 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.456664 25342 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Leader election won for term 4
00:25:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.457146 25342 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 LEADER]: Becoming Leader. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Running, Role: LEADER
00:25:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.457645 25342 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:02.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:02.461036 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 reported cstate change: term changed from 3 to 4, leader changed from 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131) to a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130). New cstate: current_term: 4 leader_uuid: "a24b7a1608504a7eb4e2421899b73f63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: HEALTHY } } }
00:25:02.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:02.928583 24408 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 -> Peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Couldn't send request to peer 24637a1459374c63935f5fc8cd1b1052. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:03.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.009810 25219 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Refusing update from remote peer a24b7a1608504a7eb4e2421899b73f63: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
00:25:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.010917 25342 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
00:25:03.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:03.298 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:03.298 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2204), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [27ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [27ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [43ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [48ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [48ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [82ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [86ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [88ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [89ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [140ms] refreshing cache from master, [140ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [143ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [145ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [147ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [147ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [559ms] refreshing cache from master, [559ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [562ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [566ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [567ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [568ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2199ms] refreshing cache from master, [2200ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [2203ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@741328879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@348eb77b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773) -> (continuation of Deferred@1698667711 after retry RPC@1314430699) -> (continuation of Deferred@153907285 after retry RPC@50923613) -> (continuation of Deferred@273165233 after retry RPC@1811224281), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8664c0e -> wakeup thread Time-limited test -> (continuation of Deferred@1713668089 after retry RPC@198809761) -> (continuation of Deferred@361179164 after retry RPC@1381807773) -> (continuation of Deferred@1698667711 after retry RPC@1314430699) -> (continuation of Deferred@153907285 after retry RPC@50923613) -> (continuation of Deferred@273165233 after retry RPC@1811224281))) after lookup
00:25:03.299 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:03.301 [DEBUG - kudu-nio-0] (Connection.java:757) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] Successfully connected to peer
00:25:03.330 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1fc9763f, L:/127.0.0.1:39240 - R:/127.23.2.130:42999] using TOKEN/null
00:25:03.357 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.131:35751
00:25:03.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.359297 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:03.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.719619 25404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:03.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.719980 25404 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:03.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.720068 25404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.727547 25404 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.727802 25404 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.131
00:25:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.737452 25404 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.738952 25404 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.741338 25404 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.756249 25416 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.756246 25417 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:03.756273 25419 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.757642 25404 server_base.cc:1034] running on GCE node
00:25:03.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.758309 25404 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.759361 25404 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:03.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.760602 25404 hybrid_clock.cc:648] HybridClock initialized: now 1786494303760483 us; error 94 us; skew 500 ppm
00:25:03.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.760958 25404 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.763513 25404 webserver.cc:460] Webserver started at http://127.23.2.131:46861/ using document root <none> and password file <none>
00:25:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.764362 25404 fs_manager.cc:359] Metadata directory not provided
00:25:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.764494 25404 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:03.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.770557 25404 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
00:25:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.773900 25434 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.775391 25404 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.775550 25404 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.776624 25404 fs_report.cc:383] FS layout report
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/data
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:03.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.810431 25404 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:03.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.811409 25404 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:03.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.811724 25404 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:03.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.812814 25404 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.814992 25442 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.821094 25404 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.821240 25404 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.821342 25404 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:03.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.824520 25404 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:03.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.824609 25404 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
00:25:03.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.825093 25442 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap starting.
00:25:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.852250 25442 log.cc:824] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Log is configured to *not* fsync() on all Append() calls
00:25:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.866657 25404 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.131:35751
00:25:03.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.866755 25661 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.131:35751 every 8 connection(s)
00:25:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.868213 25404 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:03.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.871532 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 25404
00:25:03.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.893762 25442 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.894603 25442 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap complete.
00:25:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.904655 25442 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent bootstrapping tablet: real 0.080s	user 0.039s	sys 0.011s
00:25:03.923 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:03.924 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.130:42999
00:25:03.925 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:25:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.925638 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 24380
00:25:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.920672 25442 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.947558 25442 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Initialized, Role: FOLLOWER
00:25:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.954654 25442 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:03.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.962949 25663 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:03.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.963430 25663 heartbeater.cc:464] Registering TS with master...
00:25:03.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.964746 25663 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:03.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.968446 23652 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.977979 25662 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.978222 25662 heartbeater.cc:464] Registering TS with master...
00:25:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.978825 25662 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.980376 23937 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.980604 25442 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent starting tablet: real 0.076s	user 0.014s	sys 0.000s
00:25:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.981946 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.131:53671
00:25:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.986245 25662 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.998311 25664 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.998534 25664 heartbeater.cc:464] Registering TS with master...
00:25:03.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:03.998890 25664 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:04.001307 23802 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:04.001478 24595 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:04.004190 25559 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 3 FOLLOWER]: Advancing to term 4
00:25:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:04.128970 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 24380
00:25:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:04.153074 23910 connection.cc:535] server connection from 127.23.2.130:52063 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:25:04.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:04.153 [DEBUG - kudu-nio-0] (Connection.java:239) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] handling channelInactive
00:25:04.154 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state READY due to: connection closed
00:25:04.154 [DEBUG - Time-limited test] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:04.156 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.157 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.178 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:04.178 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:04.179 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [3ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [3ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@276179910(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2035bc85 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6453dede -> wakeup thread Time-limited test)) after lookup
00:25:04.179 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:04.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.182 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.196 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:04.197 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:04.197 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [3ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [3ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [27ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [28ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@276179910(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2035bc85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6453dede -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903))) after lookup
00:25:04.198 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:04.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.199 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.238 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:04.239 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:04.239 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [3ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [3ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [27ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [28ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [45ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [45ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [84ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@276179910(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2035bc85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6453dede -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674))) after lookup
00:25:04.240 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:04.242 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.243 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.357 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:04.357 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:04.358 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=204), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [3ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [3ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [27ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [28ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [45ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [45ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [84ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [86ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [88ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [89ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [199ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [203ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@276179910(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2035bc85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674) -> (continuation of Deferred@37662746 after retry RPC@904067533), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6453dede -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674) -> (continuation of Deferred@37662746 after retry RPC@904067533))) after lookup
00:25:04.359 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:04.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.360 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:04.458 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:04.458 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=304), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [3ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [3ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [27ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [28ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [45ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [45ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [84ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [86ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [88ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [89ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [199ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [203ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [204ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [206ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [206ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [300ms] refreshing cache from master, [300ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [303ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@276179910(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2035bc85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674) -> (continuation of Deferred@37662746 after retry RPC@904067533) -> (continuation of Deferred@1697018641 after retry RPC@988141947), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6453dede -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674) -> (continuation of Deferred@37662746 after retry RPC@904067533) -> (continuation of Deferred@1697018641 after retry RPC@988141947))) after lookup
00:25:04.459 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:04.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:04.461 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.296679 25697 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.296940 25697 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.298556 25697 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:05.300601 25093 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:05.302830 25093 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.305748 25559 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:25:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.306882 25093 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5635f94622f74554984e41143786b602; no voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63
00:25:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.307325 25697 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
00:25:05.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.519306 25709 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:05.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.519629 25709 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.521443 25709 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:05.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:05.523303 25437 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:05.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:05.525240 25437 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.529911 25219 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602" is_pre_election: true
00:25:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.530191 25219 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 4.
00:25:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.530807 25436 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: a24b7a1608504a7eb4e2421899b73f63
00:25:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.531235 25709 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 4 FOLLOWER]: Leader pre-election won for term 5
00:25:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.531378 25709 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 4 FOLLOWER]: Starting leader election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.531489 25709 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 4 FOLLOWER]: Advancing to term 5
00:25:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.533682 25709 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.534427 25709 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 5 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.535192 25219 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602"
00:25:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.535454 25219 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 4 FOLLOWER]: Advancing to term 5
00:25:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:05.536561 25437 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.537868 25219 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 5.
00:25:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.538335 25436 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: a24b7a1608504a7eb4e2421899b73f63
00:25:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.538672 25709 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Leader election won for term 5
00:25:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.539170 25709 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 LEADER]: Becoming Leader. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Running, Role: LEADER
00:25:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.539749 25709 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:05.543819 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 reported cstate change: term changed from 4 to 5, leader changed from a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130) to 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131). New cstate: current_term: 5 leader_uuid: "24637a1459374c63935f5fc8cd1b1052" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:25:06.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.031564 25219 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Refusing update from remote peer 24637a1459374c63935f5fc8cd1b1052: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
00:25:06.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.032474 25709 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
00:25:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.043082 25437 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 -> Peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Couldn't send request to peer a24b7a1608504a7eb4e2421899b73f63. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:06.258 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:06.258 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:06.259 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2105), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [3ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [3ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [23ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [25ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [27ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [28ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [45ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [45ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [84ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [86ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [88ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [89ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [199ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [203ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [204ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [206ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [206ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [300ms] refreshing cache from master, [300ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [303ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [305ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [307ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [307ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2100ms] refreshing cache from master, [2100ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [2103ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@276179910(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2035bc85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674) -> (continuation of Deferred@37662746 after retry RPC@904067533) -> (continuation of Deferred@1697018641 after retry RPC@988141947) -> (continuation of Deferred@1003114226 after retry RPC@804288241), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6453dede -> wakeup thread Time-limited test -> (continuation of Deferred@1262203136 after retry RPC@837034903) -> (continuation of Deferred@1489861313 after retry RPC@1660592674) -> (continuation of Deferred@37662746 after retry RPC@904067533) -> (continuation of Deferred@1697018641 after retry RPC@988141947) -> (continuation of Deferred@1003114226 after retry RPC@804288241))) after lookup
00:25:06.260 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:06.261 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] Successfully connected to peer
00:25:06.292 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xeea3f70e, L:/127.0.0.1:48106 - R:/127.23.2.131:35751] using TOKEN/null
00:25:06.333 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.130:42999
00:25:06.334 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.334975 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:42999
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44417
00:25:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:25:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:06.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.686672 25756 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:06.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.687096 25756 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.687243 25756 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.695381 25756 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.695621 25756 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.130
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.705636 25756 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:42999
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44417
00:25:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.707649 25756 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:06.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.709829 25756 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.724745 25767 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.724833 25770 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:06.724733 25765 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.729454 25756 server_base.cc:1034] running on GCE node
00:25:06.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.730235 25756 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.731314 25756 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.732571 25756 hybrid_clock.cc:648] HybridClock initialized: now 1786494306732460 us; error 74 us; skew 500 ppm
00:25:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.732890 25756 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:06.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.735409 25756 webserver.cc:460] Webserver started at http://127.23.2.130:44417/ using document root <none> and password file <none>
00:25:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.736194 25756 fs_manager.cc:359] Metadata directory not provided
00:25:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.736323 25756 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:06.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.742395 25756 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
00:25:06.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.745801 25781 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:06.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.747303 25756 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.747481 25756 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.748652 25756 fs_report.cc:383] FS layout report
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/data
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.781121 25756 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:06.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.782061 25756 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:06.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.782351 25756 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:06.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.783439 25756 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.785667 25794 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.791571 25756 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.791744 25756 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
00:25:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.791857 25756 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.795071 25756 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.795150 25756 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
00:25:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.795578 25794 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap starting.
00:25:06.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.821079 25794 log.cc:824] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Log is configured to *not* fsync() on all Append() calls
00:25:06.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.839241 25756 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.130:42999
00:25:06.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.839339 26006 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.130:42999 every 8 connection(s)
00:25:06.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.840745 25756 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.847308 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 25756
00:25:06.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.855759 25794 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.856527 25794 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap complete.
00:25:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.858142 25794 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent bootstrapping tablet: real 0.063s	user 0.035s	sys 0.015s
00:25:06.862 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:06.882 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.131:35751
00:25:06.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.883443 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 25404
00:25:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.885511 26013 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.885857 26013 heartbeater.cc:464] Registering TS with master...
00:25:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.886780 26013 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:06.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.889599 23802 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.899225 26007 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.899367 26007 heartbeater.cc:464] Registering TS with master...
00:25:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.899806 26007 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.900388 26011 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.900509 26011 heartbeater.cc:464] Registering TS with master...
00:25:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.900882 26011 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:06.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.902266 23936 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:06.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.902312 23652 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:06.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.903795 23936 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.130:41683
00:25:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.910825 25794 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.912186 25794 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Initialized, Role: FOLLOWER
00:25:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.913057 25794 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.915853 25651 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.916357 26007 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:06.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:06.919179 25794 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent starting tablet: real 0.061s	user 0.011s	sys 0.006s
00:25:07.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:07.021931 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 25404
00:25:07.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:07.050 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] handling channelInactive
00:25:07.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state READY due to: connection closed
00:25:07.051 [DEBUG - Time-limited test] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:07.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:07.051733 26039 negotiation.cc:337] Failed RPC negotiation. Trace:
00:25:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:07.028230 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.23.2.131:44191 (local address 127.23.2.130:42999)
00:25:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:07.030770 (+  2540us) server_negotiation.cc:206] Beginning negotiation
00:25:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:07.030778 (+     8us) server_negotiation.cc:399] Waiting for connection header
00:25:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:25:07.050099 (+ 19321us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.23.2.131:44191: BlockingRecv error: recv got EOF from 127.23.2.131:44191 (error 108)
00:25:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2329,"thread_start_us":2148,"threads_started":1}
00:25:07.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.053 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.078 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:07.078 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:07.078 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1095472966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@21a6dab4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f06908b -> wakeup thread Time-limited test)) after lookup
00:25:07.079 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:07.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.081 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.097 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:07.097 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:07.098 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [28ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [30ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [30ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1095472966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@21a6dab4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f06908b -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977))) after lookup
00:25:07.098 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:07.100 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.100 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.117 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:07.118 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:07.118 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [28ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [30ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [30ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [47ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [49ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [49ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1095472966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@21a6dab4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f06908b -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042))) after lookup
00:25:07.119 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:07.120 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.120 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.217 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:07.218 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:07.218 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=167), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [28ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [30ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [30ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [47ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [49ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [49ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [67ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [69ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [69ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [166ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1095472966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@21a6dab4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042) -> (continuation of Deferred@471765689 after retry RPC@657267931), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f06908b -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042) -> (continuation of Deferred@471765689 after retry RPC@657267931))) after lookup
00:25:07.219 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:07.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.221 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.676 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:07.677 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:07.677 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=627), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [28ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [30ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [30ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [47ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [49ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [49ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [67ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [69ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [69ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [166ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [168ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [170ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [170ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [622ms] refreshing cache from master, [622ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [625ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1095472966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@21a6dab4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042) -> (continuation of Deferred@471765689 after retry RPC@657267931) -> (continuation of Deferred@175921686 after retry RPC@1165513798), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f06908b -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042) -> (continuation of Deferred@471765689 after retry RPC@657267931) -> (continuation of Deferred@175921686 after retry RPC@1165513798))) after lookup
00:25:07.678 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:07.679 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:07.679 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:08.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.077955 26042 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:25:08.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.078280 26042 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.080054 26042 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:08.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:08.085160 25784 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:08.086874 25784 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:08.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.090500 25219 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602" is_pre_election: true
00:25:08.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.091336 25783 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602
00:25:08.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.091761 26042 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
00:25:08.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.346553 26049 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:08.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.346771 26049 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.347378 26049 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:08.350538 25093 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:08.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.356535 25921 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:25:08.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.357017 25921 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 4.
00:25:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.358105 25093 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.358439 26049 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Leader pre-election won for term 6
00:25:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.358549 26049 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.358613 26049 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 5 FOLLOWER]: Advancing to term 6
00:25:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.360667 26049 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:08.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.361358 26049 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 6 election: Requested vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:08.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.362183 25921 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:25:08.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.362439 25921 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 4 FOLLOWER]: Advancing to term 6
00:25:08.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:08.362800 25093 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:08.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.366693 25921 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 6.
00:25:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.367162 25093 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.367481 26049 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Leader election won for term 6
00:25:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.367980 26049 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 LEADER]: Becoming Leader. State: Replica: 5635f94622f74554984e41143786b602, State: Running, Role: LEADER
00:25:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.368491 26049 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.371881 23936 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 reported cstate change: term changed from 5 to 6, leader changed from 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131) to 5635f94622f74554984e41143786b602 (127.23.2.129). New cstate: current_term: 6 leader_uuid: "5635f94622f74554984e41143786b602" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:25:08.697 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:08.698 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:08.698 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1647), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [28ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [30ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [30ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [47ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [49ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [49ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [67ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [69ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [69ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [166ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [168ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [170ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [170ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [622ms] refreshing cache from master, [622ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [625ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [627ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [628ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [628ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [1643ms] refreshing cache from master, [1643ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [1646ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1095472966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@21a6dab4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042) -> (continuation of Deferred@471765689 after retry RPC@657267931) -> (continuation of Deferred@175921686 after retry RPC@1165513798) -> (continuation of Deferred@1353532185 after retry RPC@931336376), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f06908b -> wakeup thread Time-limited test -> (continuation of Deferred@1131435883 after retry RPC@359708977) -> (continuation of Deferred@624013284 after retry RPC@899167042) -> (continuation of Deferred@471765689 after retry RPC@657267931) -> (continuation of Deferred@175921686 after retry RPC@1165513798) -> (continuation of Deferred@1353532185 after retry RPC@931336376))) after lookup
00:25:08.700 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:08.702 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] Successfully connected to peer
00:25:08.722 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6eb3bb91, L:/127.0.0.1:56246 - R:/127.23.2.129:44273] using TOKEN/null
00:25:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.729311 25921 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Refusing update from remote peer 5635f94622f74554984e41143786b602: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 18. (index mismatch)
00:25:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:08.729458 25093 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 -> Peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Couldn't send request to peer 24637a1459374c63935f5fc8cd1b1052. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.730278 26059 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
00:25:08.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.742962 26080 mvcc.cc:205] Tried to move back new op lower bound from 7317480688545181696 to 7317480687081070592. Current Snapshot: MvccSnapshot[applied={T|T < 7317480688545181696}]
00:25:08.764 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.131:35751
00:25:08.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:08.765664 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.141012 26101 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.141407 26101 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.141536 26101 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:09.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.149358 26101 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:09.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.149559 26101 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.131
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.159425 26101 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.160948 26101 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.162981 26101 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:09.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.177505 26110 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:09.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.177448 26111 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:09.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.177415 26113 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:09.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.178700 26101 server_base.cc:1034] running on GCE node
00:25:09.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.179384 26101 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:09.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.180441 26101 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.181692 26101 hybrid_clock.cc:648] HybridClock initialized: now 1786494309181559 us; error 104 us; skew 500 ppm
00:25:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.182008 26101 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.189108 26101 webserver.cc:460] Webserver started at http://127.23.2.131:46861/ using document root <none> and password file <none>
00:25:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.189864 26101 fs_manager.cc:359] Metadata directory not provided
00:25:09.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.189980 26101 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.195851 26101 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
00:25:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.198971 26121 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.200517 26101 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:25:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.200692 26101 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.201905 26101 fs_report.cc:383] FS layout report
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/data
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:09.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.220109 26101 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.221009 26101 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.221259 26101 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.222373 26101 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.224498 26129 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:09.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.230576 26101 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:09.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.230726 26101 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:09.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.230835 26101 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.234174 26101 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.234273 26101 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.001s	sys 0.000s
00:25:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.234750 26129 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap starting.
00:25:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.262825 26129 log.cc:824] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Log is configured to *not* fsync() on all Append() calls
00:25:09.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.286919 26129 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:09.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.287765 26129 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap complete.
00:25:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.288628 26101 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.131:35751
00:25:09.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.288672 26324 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.131:35751 every 8 connection(s)
00:25:09.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.289356 26129 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent bootstrapping tablet: real 0.055s	user 0.043s	sys 0.009s
00:25:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.290278 26101 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.296883 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 26101
00:25:09.297 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.308600 26129 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.310293 26129 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Initialized, Role: FOLLOWER
00:25:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.316278 26129 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.325924 26129 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent starting tablet: real 0.036s	user 0.013s	sys 0.004s
00:25:09.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.333276 26326 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:09.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.333652 26326 heartbeater.cc:464] Registering TS with master...
00:25:09.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.334640 26326 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:09.337 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:09.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.337677 23652 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:09.338 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.129:44273
00:25:09.338 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:25:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.333231 26325 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.339273 26325 heartbeater.cc:464] Registering TS with master...
00:25:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.339536 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 25061
00:25:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.339833 26325 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:09.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.342338 26333 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:09.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.342502 26333 heartbeater.cc:464] Registering TS with master...
00:25:09.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.343000 26333 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.344003 23936 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.345551 23936 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.131:44881
00:25:09.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.346935 23802 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.349206 26325 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.358075 25303 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:09.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:09.457626 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 25061
00:25:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:09.482589 25784 connection.cc:535] server connection from 127.23.2.129:51273 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:25:09.483 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] handling channelInactive
00:25:09.483 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:09.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state READY due to: connection closed
00:25:09.484 [DEBUG - Time-limited test] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:09.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.486 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:09.497 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:09.498 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1167552823(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@42ddec0f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@672fa50 -> wakeup thread Time-limited test)) after lookup
00:25:09.498 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:09.500 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.500 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.500 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.516 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:09.517 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:09.517 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [14ms] sending RPC to server 5635f94622f74554984e41143786b602, [16ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [17ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1167552823(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@42ddec0f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@672fa50 -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557))) after lookup
00:25:09.518 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:09.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.520 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.537 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:09.538 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:09.538 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [14ms] sending RPC to server 5635f94622f74554984e41143786b602, [16ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [17ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [34ms] sending RPC to server 5635f94622f74554984e41143786b602, [35ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1167552823(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@42ddec0f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@672fa50 -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812))) after lookup
00:25:09.538 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:09.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.540 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.617 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:09.617 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:09.618 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=133), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [14ms] sending RPC to server 5635f94622f74554984e41143786b602, [16ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [17ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [34ms] sending RPC to server 5635f94622f74554984e41143786b602, [35ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [54ms] sending RPC to server 5635f94622f74554984e41143786b602, [56ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [57ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [132ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1167552823(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@42ddec0f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812) -> (continuation of Deferred@1371451652 after retry RPC@2110008334), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@672fa50 -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812) -> (continuation of Deferred@1371451652 after retry RPC@2110008334))) after lookup
00:25:09.618 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:09.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.620 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.757 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:09.757 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:09.778 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=273), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [14ms] sending RPC to server 5635f94622f74554984e41143786b602, [16ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [17ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [34ms] sending RPC to server 5635f94622f74554984e41143786b602, [35ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [54ms] sending RPC to server 5635f94622f74554984e41143786b602, [56ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [57ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [132ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [134ms] sending RPC to server 5635f94622f74554984e41143786b602, [136ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [136ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [272ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1167552823(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@42ddec0f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812) -> (continuation of Deferred@1371451652 after retry RPC@2110008334) -> (continuation of Deferred@1388907416 after retry RPC@357800941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@672fa50 -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812) -> (continuation of Deferred@1371451652 after retry RPC@2110008334) -> (continuation of Deferred@1388907416 after retry RPC@357800941))) after lookup
00:25:09.779 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:09.781 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.782 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:09.782 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.904726 26350 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:25:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.905027 26350 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.906708 26350 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:10.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:10.908285 26123 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:10.909955 26123 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:10.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.913851 25921 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:25:10.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.914711 26124 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052; no voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63
00:25:10.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.915124 26350 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
00:25:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.958349 26360 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 5635f94622f74554984e41143786b602)
00:25:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.958567 26360 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.959309 26360 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:10.961789 25783 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.969398 26242 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:25:10.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.969911 26242 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 5.
00:25:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.971287 25784 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:25:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.971696 26360 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Leader pre-election won for term 7
00:25:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.971853 26360 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 5635f94622f74554984e41143786b602)
00:25:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.971961 26360 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 6 FOLLOWER]: Advancing to term 7
00:25:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.974237 26360 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.975032 26360 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 7 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.975775 26242 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052"
00:25:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.976048 26242 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 5 FOLLOWER]: Advancing to term 7
00:25:10.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:10.976986 25783 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.980722 26242 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 7.
00:25:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.981271 25784 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:25:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.981626 26360 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 FOLLOWER]: Leader election won for term 7
00:25:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.982159 26360 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 LEADER]: Becoming Leader. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Running, Role: LEADER
00:25:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.982752 26360 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:10.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:10.986754 23936 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 reported cstate change: term changed from 6 to 7, leader changed from 5635f94622f74554984e41143786b602 (127.23.2.129) to a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130). New cstate: current_term: 7 leader_uuid: "a24b7a1608504a7eb4e2421899b73f63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: HEALTHY } } }
00:25:11.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.482483 26242 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Refusing update from remote peer a24b7a1608504a7eb4e2421899b73f63: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
00:25:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.483291 26360 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [LEADER]: Connected to new peer: Peer: permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
00:25:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.501912 25783 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 -> Peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Couldn't send request to peer 5635f94622f74554984e41143786b602. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:11.516 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:11.517 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:11.517 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2033), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [14ms] sending RPC to server 5635f94622f74554984e41143786b602, [16ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [17ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [32ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [34ms] sending RPC to server 5635f94622f74554984e41143786b602, [35ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [54ms] sending RPC to server 5635f94622f74554984e41143786b602, [56ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [57ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [132ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [134ms] sending RPC to server 5635f94622f74554984e41143786b602, [136ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [136ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [272ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [295ms] sending RPC to server 5635f94622f74554984e41143786b602, [298ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [298ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2029ms] refreshing cache from master, [2029ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [2032ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1167552823(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@42ddec0f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812) -> (continuation of Deferred@1371451652 after retry RPC@2110008334) -> (continuation of Deferred@1388907416 after retry RPC@357800941) -> (continuation of Deferred@344609222 after retry RPC@487537737), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@672fa50 -> wakeup thread Time-limited test -> (continuation of Deferred@1882288673 after retry RPC@1627436557) -> (continuation of Deferred@1493553734 after retry RPC@1641656812) -> (continuation of Deferred@1371451652 after retry RPC@2110008334) -> (continuation of Deferred@1388907416 after retry RPC@357800941) -> (continuation of Deferred@344609222 after retry RPC@487537737))) after lookup
00:25:11.518 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:11.518 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] Successfully connected to peer
00:25:11.537 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x87a64979, L:/127.0.0.1:38604 - R:/127.23.2.130:42999] using TOKEN/null
00:25:11.564 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.129:44273
00:25:11.565 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:25:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.566583 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:25:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:44273
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35549
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.926163 26412 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.926503 26412 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.926631 26412 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.934244 26412 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.934481 26412 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.129
00:25:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.944542 26412 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:44273
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35549
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.946321 26412 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:11.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.948316 26412 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.963411 26421 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.963443 26420 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:11.963366 26423 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.964335 26412 server_base.cc:1034] running on GCE node
00:25:11.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.965116 26412 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.966325 26412 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.967586 26412 hybrid_clock.cc:648] HybridClock initialized: now 1786494311967508 us; error 45 us; skew 500 ppm
00:25:11.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.968024 26412 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.970543 26412 webserver.cc:460] Webserver started at http://127.23.2.129:35549/ using document root <none> and password file <none>
00:25:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.971258 26412 fs_manager.cc:359] Metadata directory not provided
00:25:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.971366 26412 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.977056 26412 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
00:25:11.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.980182 26435 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.981611 26412 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:25:11.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.981808 26412 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5635f94622f74554984e41143786b602"
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:11.982911 26412 fs_report.cc:383] FS layout report
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/data
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.012280 26412 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:12.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.013289 26412 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:12.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.013636 26412 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:12.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.014794 26412 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.017230 26446 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.024770 26412 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.024930 26412 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
00:25:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.025061 26412 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.028533 26412 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.028633 26412 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
00:25:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.029065 26446 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap starting.
00:25:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.061304 26446 log.cc:824] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Log is configured to *not* fsync() on all Append() calls
00:25:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.082021 26412 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.129:44273
00:25:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.082055 26626 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.129:44273 every 8 connection(s)
00:25:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.083462 26412 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:12.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.087843 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 26412
00:25:12.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:12.094 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:97)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
00:25:12.096 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:79)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
00:25:12.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.132889 26446 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:12.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.134310 26446 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap complete.
00:25:12.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.135977 26446 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent bootstrapping tablet: real 0.107s	user 0.040s	sys 0.028s
00:25:12.136 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:12.137 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.130:42999
00:25:12.137 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:25:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.137871 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 25756
00:25:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.152779 26630 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.153164 26630 heartbeater.cc:464] Registering TS with master...
00:25:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.153631 26000 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.154314 26630 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.158998 23806 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.160111 26628 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.160295 26628 heartbeater.cc:464] Registering TS with master...
00:25:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.160742 26628 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.163018 23936 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.164613 23936 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.129:42929
00:25:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.165408 26629 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.165571 26629 heartbeater.cc:464] Registering TS with master...
00:25:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.166164 26629 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:12.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.165779 26446 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:12.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.167503 26446 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5635f94622f74554984e41143786b602, State: Initialized, Role: FOLLOWER
00:25:12.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.168545 26446 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:12.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.171417 23652 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.173763 26628 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.174773 26446 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent starting tablet: real 0.039s	user 0.011s	sys 0.007s
00:25:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:12.272753 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 25756
00:25:12.303 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] handling channelInactive
00:25:12.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:12.304 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state READY due to: connection closed
00:25:12.304 [DEBUG - Time-limited test] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:12.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.306 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.316 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:12.316 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:12.317 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1772527125(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@14a008bc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e84bc12 -> wakeup thread Time-limited test)) after lookup
00:25:12.317 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:12.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.318 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.337 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:12.338 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:12.338 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [13ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [14ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [14ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1772527125(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@14a008bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e84bc12 -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359))) after lookup
00:25:12.339 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:12.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.341 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:12.377 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:12.378 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [13ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [14ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [14ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [35ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [37ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [37ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1772527125(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@14a008bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e84bc12 -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917))) after lookup
00:25:12.378 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:12.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.379 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:12.458 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:12.458 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [13ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [14ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [14ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [35ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [37ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [37ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [74ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [75ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [75ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1772527125(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@14a008bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917) -> (continuation of Deferred@1553823607 after retry RPC@259218191), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e84bc12 -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917) -> (continuation of Deferred@1553823607 after retry RPC@259218191))) after lookup
00:25:12.459 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:12.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.461 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.917 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:12.918 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:12.918 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=614), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [13ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [14ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [14ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [35ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [37ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [37ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [74ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [75ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [75ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [155ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [157ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [157ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [610ms] refreshing cache from master, [610ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [613ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1772527125(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@14a008bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917) -> (continuation of Deferred@1553823607 after retry RPC@259218191) -> (continuation of Deferred@893618253 after retry RPC@553457353), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e84bc12 -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917) -> (continuation of Deferred@1553823607 after retry RPC@259218191) -> (continuation of Deferred@893618253 after retry RPC@553457353))) after lookup
00:25:12.919 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:12.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:12.920 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.422650 26647 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:25:13.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.423007 26647 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.424823 26647 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:13.426940 26439 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:13.428815 26439 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.432785 26242 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:25:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.433630 26439 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5635f94622f74554984e41143786b602; no voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63
00:25:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.434057 26647 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
00:25:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.525719 26657 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.525931 26657 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.526602 26657 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:13.529066 26124 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.534714 26544 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602" is_pre_election: true
00:25:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.535197 26544 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 6.
00:25:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.536350 26123 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: a24b7a1608504a7eb4e2421899b73f63
00:25:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.536671 26657 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Leader pre-election won for term 8
00:25:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.536806 26657 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Starting leader election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.536904 26657 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 7 FOLLOWER]: Advancing to term 8
00:25:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.539026 26657 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.539811 26657 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 8 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.540364 26544 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602"
00:25:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.540604 26544 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 6 FOLLOWER]: Advancing to term 8
00:25:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:13.541425 26124 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:13.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.545161 26544 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 8.
00:25:13.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.545674 26123 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: a24b7a1608504a7eb4e2421899b73f63
00:25:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.546006 26657 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 FOLLOWER]: Leader election won for term 8
00:25:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.546502 26657 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 LEADER]: Becoming Leader. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Running, Role: LEADER
00:25:13.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.547030 26657 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.551196 23936 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 reported cstate change: term changed from 7 to 8, leader changed from a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130) to 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131). New cstate: current_term: 8 leader_uuid: "24637a1459374c63935f5fc8cd1b1052" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:25:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.946498 26544 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Refusing update from remote peer 24637a1459374c63935f5fc8cd1b1052: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
00:25:13.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:13.947425 26657 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
00:25:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:13.961542 26124 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 -> Peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Couldn't send request to peer a24b7a1608504a7eb4e2421899b73f63. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:14.077 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:14.077 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:14.077 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1773), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [13ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [14ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [14ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [35ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [37ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [37ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [72ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [74ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [75ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [75ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [155ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [157ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [157ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [610ms] refreshing cache from master, [610ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [613ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [615ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [616ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [617ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [1769ms] refreshing cache from master, [1769ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [1772ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1772527125(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@14a008bc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917) -> (continuation of Deferred@1553823607 after retry RPC@259218191) -> (continuation of Deferred@893618253 after retry RPC@553457353) -> (continuation of Deferred@641316749 after retry RPC@1569526728), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e84bc12 -> wakeup thread Time-limited test -> (continuation of Deferred@1830773154 after retry RPC@1523486359) -> (continuation of Deferred@1570903576 after retry RPC@1932955917) -> (continuation of Deferred@1553823607 after retry RPC@259218191) -> (continuation of Deferred@893618253 after retry RPC@553457353) -> (continuation of Deferred@641316749 after retry RPC@1569526728))) after lookup
00:25:14.078 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:14.080 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] Successfully connected to peer
00:25:14.103 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb69a29eb, L:/127.0.0.1:59268 - R:/127.23.2.131:35751] using TOKEN/null
00:25:14.125 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.130:42999
00:25:14.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.126811 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:42999
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44417
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:25:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:14.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:14.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:14.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.480832 26714 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.481144 26714 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.481225 26714 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.489015 26714 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.489210 26714 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.130
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.498944 26714 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:42999
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44417
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.500425 26714 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.502388 26714 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.516336 26722 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.516279 26725 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:14.516279 26723 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:14.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.517194 26714 server_base.cc:1034] running on GCE node
00:25:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.517899 26714 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:14.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.518926 26714 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.520149 26714 hybrid_clock.cc:648] HybridClock initialized: now 1786494314520067 us; error 48 us; skew 500 ppm
00:25:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.520493 26714 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:14.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.525712 26714 webserver.cc:460] Webserver started at http://127.23.2.130:44417/ using document root <none> and password file <none>
00:25:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.526526 26714 fs_manager.cc:359] Metadata directory not provided
00:25:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.526656 26714 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.532320 26714 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
00:25:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.535444 26734 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.536942 26714 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.537132 26714 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.538386 26714 fs_report.cc:383] FS layout report
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/data
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.570787 26714 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.571715 26714 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.571986 26714 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.573029 26714 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.575212 26743 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:14.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.581593 26714 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:14.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.581745 26714 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:14.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.581856 26714 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:14.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.585268 26714 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:14.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.585376 26714 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
00:25:14.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.585848 26743 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap starting.
00:25:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.612417 26743 log.cc:824] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Log is configured to *not* fsync() on all Append() calls
00:25:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.637789 26714 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.130:42999
00:25:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.637857 26947 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.130:42999 every 8 connection(s)
00:25:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.639271 26714 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.647039 26743 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.647948 26743 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap complete.
00:25:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.648272 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 26714
00:25:14.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.655635 26743 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent bootstrapping tablet: real 0.070s	user 0.058s	sys 0.004s
00:25:14.666 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:14.667 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.131:35751
00:25:14.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:14.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.667752 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 26101
00:25:14.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.686499 26950 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.686951 26950 heartbeater.cc:464] Registering TS with master...
00:25:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.687770 26951 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:14.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.687954 26951 heartbeater.cc:464] Registering TS with master...
00:25:14.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.688766 26951 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:14.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.692445 23652 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:14.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.696878 26950 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:14.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.698871 23937 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:14.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.700356 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.130:38251
00:25:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.700738 26743 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.702332 26743 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Initialized, Role: FOLLOWER
00:25:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.706413 26952 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.706584 26952 heartbeater.cc:464] Registering TS with master...
00:25:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.707069 26952 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:14.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.707754 26743 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.710455 23806 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.710779 26950 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:14.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.713032 26743 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent starting tablet: real 0.057s	user 0.010s	sys 0.003s
00:25:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.721807 26318 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:14.821431 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 26101
00:25:14.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:14.851 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] handling channelInactive
00:25:14.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state READY due to: connection closed
00:25:14.852 [DEBUG - Time-limited test] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:14.854 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.854 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.854 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.877 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:14.877 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:14.878 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1992745260(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@5af48a54 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f6e9e13 -> wakeup thread Time-limited test)) after lookup
00:25:14.878 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:14.880 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.880 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.896 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:14.897 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:14.897 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [26ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [28ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [28ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1992745260(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@5af48a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f6e9e13 -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598))) after lookup
00:25:14.897 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:14.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.898 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.917 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:14.917 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:14.918 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [26ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [28ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [28ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [46ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [46ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [65ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1992745260(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@5af48a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f6e9e13 -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820))) after lookup
00:25:14.918 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:14.920 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.920 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.920 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.996 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:14.997 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:14.997 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=145), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [26ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [28ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [28ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [46ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [46ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [65ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [66ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [68ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [69ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [144ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1992745260(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@5af48a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820) -> (continuation of Deferred@1636505346 after retry RPC@225608586), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f6e9e13 -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820) -> (continuation of Deferred@1636505346 after retry RPC@225608586))) after lookup
00:25:14.998 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:14.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:14.999 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:15.256 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:15.257 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:15.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=405), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [26ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [28ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [28ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [46ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [46ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [65ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [66ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [68ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [69ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [144ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [145ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [147ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [147ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [401ms] refreshing cache from master, [401ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [404ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1992745260(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@5af48a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820) -> (continuation of Deferred@1636505346 after retry RPC@225608586) -> (continuation of Deferred@723694179 after retry RPC@890246148), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f6e9e13 -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820) -> (continuation of Deferred@1636505346 after retry RPC@225608586) -> (continuation of Deferred@723694179 after retry RPC@890246148))) after lookup
00:25:15.258 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:15.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:15.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:15.260 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.068342 26978 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.068542 26978 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:16.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.069204 26978 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:16.072422 26439 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.078147 26873 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:25:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.078749 26873 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 7.
00:25:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.079911 26439 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.080257 26978 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Leader pre-election won for term 9
00:25:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.080379 26978 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.080467 26978 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 8 FOLLOWER]: Advancing to term 9
00:25:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.082636 26978 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.083346 26978 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 9 election: Requested vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.084434 26873 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:25:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.084695 26873 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 7 FOLLOWER]: Advancing to term 9
00:25:16.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:16.085151 26439 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.089093 26873 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 9.
00:25:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.089617 26439 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.089933 26978 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Leader election won for term 9
00:25:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.090389 26978 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 LEADER]: Becoming Leader. State: Replica: 5635f94622f74554984e41143786b602, State: Running, Role: LEADER
00:25:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.090926 26978 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.095412 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 reported cstate change: term changed from 8 to 9, leader changed from 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131) to 5635f94622f74554984e41143786b602 (127.23.2.129). New cstate: current_term: 9 leader_uuid: "5635f94622f74554984e41143786b602" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:25:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.480018 26873 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Refusing update from remote peer 5635f94622f74554984e41143786b602: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
00:25:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:16.481052 26978 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
00:25:16.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:16.495654 26439 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 -> Peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Couldn't send request to peer 24637a1459374c63935f5fc8cd1b1052. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:17.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:17.257 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:17.258 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2406), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [26ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [28ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [28ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [44ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [45ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [46ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [46ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [65ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [66ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [68ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [69ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [141ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [144ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [145ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [147ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [147ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [401ms] refreshing cache from master, [401ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [404ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [406ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [408ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [408ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2402ms] refreshing cache from master, [2402ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [2404ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@1992745260(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@5af48a54 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820) -> (continuation of Deferred@1636505346 after retry RPC@225608586) -> (continuation of Deferred@723694179 after retry RPC@890246148) -> (continuation of Deferred@2131829662 after retry RPC@1242871884), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f6e9e13 -> wakeup thread Time-limited test -> (continuation of Deferred@1658977855 after retry RPC@2045012598) -> (continuation of Deferred@1830946643 after retry RPC@762854820) -> (continuation of Deferred@1636505346 after retry RPC@225608586) -> (continuation of Deferred@723694179 after retry RPC@890246148) -> (continuation of Deferred@2131829662 after retry RPC@1242871884))) after lookup
00:25:17.258 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:17.259 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] Successfully connected to peer
00:25:17.280 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1e5c01fc, L:/127.0.0.1:56308 - R:/127.23.2.129:44273] using TOKEN/null
00:25:17.318 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.131:35751
00:25:17.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.318965 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:17.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.684723 27038 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:17.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.685109 27038 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:17.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.685230 27038 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:17.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.693463 27038 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:17.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.693694 27038 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.131
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.705011 27038 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.706959 27038 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.709321 27038 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.724789 27053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.724829 27054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:17.724726 27057 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.725620 27038 server_base.cc:1034] running on GCE node
00:25:17.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.726366 27038 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:17.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.727442 27038 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:17.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.728677 27038 hybrid_clock.cc:648] HybridClock initialized: now 1786494317728593 us; error 52 us; skew 500 ppm
00:25:17.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.729028 27038 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.731796 27038 webserver.cc:460] Webserver started at http://127.23.2.131:46861/ using document root <none> and password file <none>
00:25:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.732547 27038 fs_manager.cc:359] Metadata directory not provided
00:25:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.732656 27038 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.738255 27038 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
00:25:17.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.741379 27063 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:17.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.742934 27038 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.743084 27038 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.744326 27038 fs_report.cc:383] FS layout report
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/data
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.763855 27038 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.764780 27038 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:17.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.765095 27038 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:17.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.766191 27038 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:17.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.768456 27073 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.774578 27038 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.774714 27038 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.774821 27038 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:17.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.778246 27038 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:17.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.778347 27038 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
00:25:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.778852 27073 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap starting.
00:25:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.807962 27073 log.cc:824] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Log is configured to *not* fsync() on all Append() calls
00:25:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.823267 27038 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.131:35751
00:25:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.823318 27265 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.131:35751 every 8 connection(s)
00:25:17.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.824729 27038 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:17.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.832872 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 27038
00:25:17.833 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:17.854 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:17.855 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.129:44273
00:25:17.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:25:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.855645 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 26412
00:25:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.887493 26614 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.907709 27269 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:17.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.908145 27269 heartbeater.cc:464] Registering TS with master...
00:25:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.909310 27269 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.911355 27266 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.911523 27266 heartbeater.cc:464] Registering TS with master...
00:25:17.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.912031 27266 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:17.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.913254 23806 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.914134 27268 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.914284 27268 heartbeater.cc:464] Registering TS with master...
00:25:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.914657 23937 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.914821 27268 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.916239 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.131:44965
00:25:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.916373 27073 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.917090 27073 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap complete.
00:25:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.917330 23652 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.918617 27073 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent bootstrapping tablet: real 0.140s	user 0.060s	sys 0.008s
00:25:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.927317 27073 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.928738 27073 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Initialized, Role: FOLLOWER
00:25:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.929541 27073 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.930972 27266 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:17.931532 27073 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent starting tablet: real 0.013s	user 0.015s	sys 0.000s
00:25:18.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:18.007891 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 26412
00:25:18.039 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] handling channelInactive
00:25:18.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:18.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state READY due to: connection closed
00:25:18.040 [DEBUG - Time-limited test] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.042 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.042 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.042 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.056 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:18.057 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:18.057 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test)) after lookup
00:25:18.058 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.059 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.077 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:18.078 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:18.078 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [17ms] sending RPC to server 5635f94622f74554984e41143786b602, [19ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [19ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [37ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972))) after lookup
00:25:18.079 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.082 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.117 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:18.117 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:18.118 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [17ms] sending RPC to server 5635f94622f74554984e41143786b602, [19ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [19ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [37ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [39ms] sending RPC to server 5635f94622f74554984e41143786b602, [41ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [42ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [76ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353))) after lookup
00:25:18.118 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.119 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.176 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:18.177 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:18.177 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=137), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [17ms] sending RPC to server 5635f94622f74554984e41143786b602, [19ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [19ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [37ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [39ms] sending RPC to server 5635f94622f74554984e41143786b602, [41ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [42ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [76ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [78ms] sending RPC to server 5635f94622f74554984e41143786b602, [79ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [136ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829))) after lookup
00:25:18.178 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.180 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:18.298 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:18.298 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=258), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [17ms] sending RPC to server 5635f94622f74554984e41143786b602, [19ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [19ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [37ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [39ms] sending RPC to server 5635f94622f74554984e41143786b602, [41ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [42ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [76ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [78ms] sending RPC to server 5635f94622f74554984e41143786b602, [79ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [136ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [138ms] sending RPC to server 5635f94622f74554984e41143786b602, [140ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [140ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [257ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829) -> (continuation of Deferred@611041676 after retry RPC@514964637), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829) -> (continuation of Deferred@611041676 after retry RPC@514964637))) after lookup
00:25:18.299 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.301 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.301 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.301 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.617 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:18.617 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:18.618 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=577), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [17ms] sending RPC to server 5635f94622f74554984e41143786b602, [19ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [19ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [37ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [39ms] sending RPC to server 5635f94622f74554984e41143786b602, [41ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [42ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [76ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [78ms] sending RPC to server 5635f94622f74554984e41143786b602, [79ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [136ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [138ms] sending RPC to server 5635f94622f74554984e41143786b602, [140ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [140ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [257ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [259ms] sending RPC to server 5635f94622f74554984e41143786b602, [261ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [261ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [573ms] refreshing cache from master, [573ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [576ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829) -> (continuation of Deferred@611041676 after retry RPC@514964637) -> (continuation of Deferred@1603067150 after retry RPC@1129258109), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829) -> (continuation of Deferred@611041676 after retry RPC@514964637) -> (continuation of Deferred@1603067150 after retry RPC@1129258109))) after lookup
00:25:18.618 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:18.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:18.620 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5635f94622f74554984e41143786b602(127.23.2.129:44273) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273
00:25:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.211027 27294 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 5635f94622f74554984e41143786b602)
00:25:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.211241 27294 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.212806 27294 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.214011 26736 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.215512 26736 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.221909 27201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:25:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.222493 27201 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 8.
00:25:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.223651 26737 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:25:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.224164 27294 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Leader pre-election won for term 10
00:25:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.224288 27294 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 5635f94622f74554984e41143786b602)
00:25:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.224362 27294 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 9 FOLLOWER]: Advancing to term 10
00:25:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.226363 27294 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.226997 27294 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 10 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.227645 27201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052"
00:25:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.227974 27201 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 8 FOLLOWER]: Advancing to term 10
00:25:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.228724 26736 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111)
00:25:19.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.232402 27201 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24b7a1608504a7eb4e2421899b73f63 in term 10.
00:25:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.232930 26737 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63; no voters: 5635f94622f74554984e41143786b602
00:25:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.233279 27294 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Leader election won for term 10
00:25:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.233750 27294 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 LEADER]: Becoming Leader. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Running, Role: LEADER
00:25:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.234261 27294 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:19.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.238134 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 reported cstate change: term changed from 9 to 10, leader changed from 5635f94622f74554984e41143786b602 (127.23.2.129) to a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130). New cstate: current_term: 10 leader_uuid: "a24b7a1608504a7eb4e2421899b73f63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: HEALTHY } } }
00:25:19.396 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:19.397 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:19.397 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1357), Traces: [0ms] sending RPC to server 5635f94622f74554984e41143786b602, [2ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [17ms] sending RPC to server 5635f94622f74554984e41143786b602, [19ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [19ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [37ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [39ms] sending RPC to server 5635f94622f74554984e41143786b602, [41ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [42ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [76ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [78ms] sending RPC to server 5635f94622f74554984e41143786b602, [79ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [79ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [136ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [138ms] sending RPC to server 5635f94622f74554984e41143786b602, [140ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [140ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [254ms] refreshing cache from master, [254ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [257ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [259ms] sending RPC to server 5635f94622f74554984e41143786b602, [261ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [261ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [573ms] refreshing cache from master, [573ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [576ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [578ms] sending RPC to server 5635f94622f74554984e41143786b602, [580ms] received response from server 5635f94622f74554984e41143786b602: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [580ms] delaying RPC due to: Network error: Failed to connect to peer 5635f94622f74554984e41143786b602(127.23.2.129:44273): Connection refused: /127.23.2.129:44273, [1353ms] refreshing cache from master, [1353ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [1356ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@38731389(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@595c35ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829) -> (continuation of Deferred@611041676 after retry RPC@514964637) -> (continuation of Deferred@1603067150 after retry RPC@1129258109) -> (continuation of Deferred@1503797644 after retry RPC@1647665713), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a8e61f7 -> wakeup thread Time-limited test -> (continuation of Deferred@2078563084 after retry RPC@1377692972) -> (continuation of Deferred@1889629676 after retry RPC@1307083353) -> (continuation of Deferred@859511946 after retry RPC@1542205829) -> (continuation of Deferred@611041676 after retry RPC@514964637) -> (continuation of Deferred@1603067150 after retry RPC@1129258109) -> (continuation of Deferred@1503797644 after retry RPC@1647665713))) after lookup
00:25:19.398 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:19.399 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] Successfully connected to peer
00:25:19.419 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x337e579a, L:/127.0.0.1:51400 - R:/127.23.2.130:42999] using TOKEN/null
00:25:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.425206 27201 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Refusing update from remote peer a24b7a1608504a7eb4e2421899b73f63: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 34. (index mismatch)
00:25:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.425307 26736 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 -> Peer 5635f94622f74554984e41143786b602 (127.23.2.129:44273): Couldn't send request to peer 5635f94622f74554984e41143786b602. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.129:44273: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.426180 27310 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [LEADER]: Connected to new peer: Peer: permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
00:25:19.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.438405 27333 mvcc.cc:205] Tried to move back new op lower bound from 7317480732354744320 to 7317480731587567616. Current Snapshot: MvccSnapshot[applied={T|T < 7317480732354744320}]
00:25:19.455 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.129:44273
00:25:19.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.456889 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:44273
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35549
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.813117 27358 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.813484 27358 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.813609 27358 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.821255 27358 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.821494 27358 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.129
00:25:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.831413 27358 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:44273
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35549
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/logs
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.833446 27358 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.835587 27358 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:19.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.850734 27366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:19.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.850703 27367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:19.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:19.850672 27370 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.852003 27358 server_base.cc:1034] running on GCE node
00:25:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.852777 27358 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:19.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.853919 27358 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:19.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.855185 27358 hybrid_clock.cc:648] HybridClock initialized: now 1786494319855106 us; error 42 us; skew 500 ppm
00:25:19.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.855587 27358 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:19.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.861440 27358 webserver.cc:460] Webserver started at http://127.23.2.129:35549/ using document root <none> and password file <none>
00:25:19.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.862389 27358 fs_manager.cc:359] Metadata directory not provided
00:25:19.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.862538 27358 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.868592 27358 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.003s
00:25:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.871989 27378 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.873576 27358 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.873800 27358 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5635f94622f74554984e41143786b602"
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.874948 27358 fs_report.cc:383] FS layout report
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/wal
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/data
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.895730 27358 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.896752 27358 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.897094 27358 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.898286 27358 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.900626 27391 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.906852 27358 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.907040 27358 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
00:25:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.907181 27358 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:19.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.910663 27358 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:19.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.910768 27358 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.000s	sys 0.001s
00:25:19.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.911288 27391 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap starting.
00:25:19.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.941553 27391 log.cc:824] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Log is configured to *not* fsync() on all Append() calls
00:25:19.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.964046 27358 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.129:44273
00:25:19.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.964063 27601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.129:44273 every 8 connection(s)
00:25:19.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.965497 27358 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-0/data/info.pb
00:25:19.969 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:19.968640 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 27358
00:25:20.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.001183 27391 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.002023 27391 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Bootstrap complete.
00:25:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.003522 27391 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent bootstrapping tablet: real 0.093s	user 0.058s	sys 0.016s
00:25:20.004 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:20.005 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.130:42999
00:25:20.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:25:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.006425 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 26714
00:25:20.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.014951 27391 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.023000 27391 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5635f94622f74554984e41143786b602, State: Initialized, Role: FOLLOWER
00:25:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.024878 27391 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.027776 27610 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.028019 27602 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.028218 27602 heartbeater.cc:464] Registering TS with master...
00:25:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.028410 27603 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.028541 27603 heartbeater.cc:464] Registering TS with master...
00:25:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.029500 27603 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.029956 27602 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.030799 27610 heartbeater.cc:464] Registering TS with master...
00:25:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.031371 27610 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:20.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.031862 27391 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602: Time spent starting tablet: real 0.028s	user 0.014s	sys 0.005s
00:25:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.032994 23937 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.033048 23653 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.034579 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.129:59415
00:25:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.036178 23806 ts_manager.cc:194] Re-registered known tserver with Master: 5635f94622f74554984e41143786b602 (127.23.2.129:44273)
00:25:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.041495 27602 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.048553 26942 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:20.148656 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 26714
00:25:20.179 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] handling channelInactive
00:25:20.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:20.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state READY due to: connection closed
00:25:20.180 [DEBUG - Time-limited test] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.182 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.197 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:20.197 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:20.197 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test)) after lookup
00:25:20.198 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.199 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.216 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:20.217 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:20.217 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [18ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [19ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [36ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463))) after lookup
00:25:20.218 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.220 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.220 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.237 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:20.237 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:20.238 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [18ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [19ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [36ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [38ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [40ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [40ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [57ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927))) after lookup
00:25:20.238 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.239 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.239 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.336 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:20.336 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:20.337 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=156), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [18ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [19ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [36ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [38ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [40ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [40ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [57ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [58ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [59ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [59ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [156ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731))) after lookup
00:25:20.338 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.340 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:20.497 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:20.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=317), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [18ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [19ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [36ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [38ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [40ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [40ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [57ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [58ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [59ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [59ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [156ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [157ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [160ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [160ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [314ms] refreshing cache from master, [314ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [316ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731) -> (continuation of Deferred@1957762318 after retry RPC@514474161), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731) -> (continuation of Deferred@1957762318 after retry RPC@514474161))) after lookup
00:25:20.498 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.500 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.976 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:20.977 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)[L]]}]
00:25:20.977 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=797), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [18ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [19ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [36ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [38ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [40ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [40ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [57ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [58ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [59ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [59ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [156ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [157ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [160ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [160ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [314ms] refreshing cache from master, [314ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [316ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [318ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [320ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [320ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [793ms] refreshing cache from master, [794ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [796ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731) -> (continuation of Deferred@1957762318 after retry RPC@514474161) -> (continuation of Deferred@1013466413 after retry RPC@1759628681), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731) -> (continuation of Deferred@1957762318 after retry RPC@514474161) -> (continuation of Deferred@1013466413 after retry RPC@1759628681))) after lookup
00:25:20.978 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] connecting to peer
00:25:20.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] cleaning up while in state CONNECTING due to: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:20.980 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999
00:25:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.355342 27630 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:25:21.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.355655 27630 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.357384 27630 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:21.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:21.359575 27385 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:21.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:21.361362 27385 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:21.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.365656 27201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "24637a1459374c63935f5fc8cd1b1052" is_pre_election: true
00:25:21.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.366644 27385 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5635f94622f74554984e41143786b602; no voters: 24637a1459374c63935f5fc8cd1b1052, a24b7a1608504a7eb4e2421899b73f63
00:25:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.367070 27630 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
00:25:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.442400 27637 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.442610 27637 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:21.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.444396 27637 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:21.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:21.448391 27067 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:21.450502 27067 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.455129 27510 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602" is_pre_election: true
00:25:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.455624 27510 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 9.
00:25:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.456830 27065 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: a24b7a1608504a7eb4e2421899b73f63
00:25:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.457234 27637 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Leader pre-election won for term 11
00:25:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.457382 27637 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Starting leader election (detected failure of leader a24b7a1608504a7eb4e2421899b73f63)
00:25:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.457485 27637 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 10 FOLLOWER]: Advancing to term 11
00:25:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.459527 27637 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.460240 27637 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 11 election: Requested vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.460789 27510 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "24637a1459374c63935f5fc8cd1b1052" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602"
00:25:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.461026 27510 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 9 FOLLOWER]: Advancing to term 11
00:25:21.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:21.462212 27067 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111)
00:25:21.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.465478 27510 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24637a1459374c63935f5fc8cd1b1052 in term 11.
00:25:21.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.465952 27065 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602; no voters: a24b7a1608504a7eb4e2421899b73f63
00:25:21.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.466331 27637 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 11 FOLLOWER]: Leader election won for term 11
00:25:21.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.466949 27637 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 11 LEADER]: Becoming Leader. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Running, Role: LEADER
00:25:21.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.467485 27637 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.471341 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 reported cstate change: term changed from 10 to 11, leader changed from a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130) to 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131). New cstate: current_term: 11 leader_uuid: "24637a1459374c63935f5fc8cd1b1052" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:25:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.875422 27510 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Refusing update from remote peer 24637a1459374c63935f5fc8cd1b1052: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
00:25:21.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:21.876375 27637 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
00:25:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:21.892485 27067 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 -> Peer a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999): Couldn't send request to peer a24b7a1608504a7eb4e2421899b73f63. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.130:42999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:22.516 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:22.517 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:22.517 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2337), Traces: [0ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [2ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [16ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [18ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [19ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [19ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [36ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [38ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [40ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [40ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [57ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [58ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [59ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [59ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [156ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [157ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [160ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [160ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [314ms] refreshing cache from master, [314ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [316ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [318ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [320ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [320ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [793ms] refreshing cache from master, [794ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [796ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [798ms] sending RPC to server a24b7a1608504a7eb4e2421899b73f63, [800ms] received response from server a24b7a1608504a7eb4e2421899b73f63: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [800ms] delaying RPC due to: Network error: Failed to connect to peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999): Connection refused: /127.23.2.130:42999, [2333ms] refreshing cache from master, [2333ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [2336ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@801665855(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@2c2d34cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731) -> (continuation of Deferred@1957762318 after retry RPC@514474161) -> (continuation of Deferred@1013466413 after retry RPC@1759628681) -> (continuation of Deferred@1101647532 after retry RPC@667078293), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41d1cc7c -> wakeup thread Time-limited test -> (continuation of Deferred@108404671 after retry RPC@541860463) -> (continuation of Deferred@900773260 after retry RPC@1957606927) -> (continuation of Deferred@1672602023 after retry RPC@1734451731) -> (continuation of Deferred@1957762318 after retry RPC@514474161) -> (continuation of Deferred@1013466413 after retry RPC@1759628681) -> (continuation of Deferred@1101647532 after retry RPC@667078293))) after lookup
00:25:22.518 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:22.519 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] Successfully connected to peer
00:25:22.535 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6ba6ad2e, L:/127.0.0.1:44000 - R:/127.23.2.131:35751] using TOKEN/null
00:25:22.560 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.130:42999
00:25:22.561 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.561877 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:42999
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44417
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.917870 27697 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.918205 27697 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.918304 27697 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.925980 27697 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.926182 27697 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.130
00:25:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.936209 27697 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:42999
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44417
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/logs
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.937676 27697 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.939648 27697 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.954504 27705 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.954833 27713 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:22.954478 27706 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.955742 27697 server_base.cc:1034] running on GCE node
00:25:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.956394 27697 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.957420 27697 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:22.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.958673 27697 hybrid_clock.cc:648] HybridClock initialized: now 1786494322958580 us; error 50 us; skew 500 ppm
00:25:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.958977 27697 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:22.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.961683 27697 webserver.cc:460] Webserver started at http://127.23.2.130:44417/ using document root <none> and password file <none>
00:25:22.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.962414 27697 fs_manager.cc:359] Metadata directory not provided
00:25:22.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.962520 27697 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.968542 27697 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.003s
00:25:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.971794 27723 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.973369 27697 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
00:25:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.973587 27697 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:25:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:53 on dist-test-slave-6bbx"
00:25:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.974687 27697 fs_report.cc:383] FS layout report
00:25:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/wal
00:25:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/data
00:25:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.987962 27697 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.988988 27697 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.989334 27697 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.990495 27697 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:22.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.992826 27731 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.999037 27697 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.999208 27697 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:22.999361 27697 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.003036 27697 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.003158 27697 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.000s	sys 0.001s
00:25:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.003716 27731 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap starting.
00:25:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.034515 27731 log.cc:824] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Log is configured to *not* fsync() on all Append() calls
00:25:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.043403 27697 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.130:42999
00:25:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.043423 27913 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.130:42999 every 8 connection(s)
00:25:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.045174 27697 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-1/data/info.pb
00:25:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.052380 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 27697
00:25:23.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.088634 27731 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:23.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.089541 27731 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Bootstrap complete.
00:25:23.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.091428 27731 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent bootstrapping tablet: real 0.088s	user 0.069s	sys 0.013s
00:25:23.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.103313 27731 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:23.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.104988 27731 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24b7a1608504a7eb4e2421899b73f63, State: Initialized, Role: FOLLOWER
00:25:23.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.106151 27731 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:23.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.117651 27731 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63: Time spent starting tablet: real 0.026s	user 0.016s	sys 0.010s
00:25:23.119 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.119575 27915 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:23.119 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.2.131:35751
00:25:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.120042 27915 heartbeater.cc:464] Registering TS with master...
00:25:23.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.121250 27915 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.122411 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 27038
00:25:23.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.124920 23653 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.123133 27914 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.128355 27914 heartbeater.cc:464] Registering TS with master...
00:25:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.124648 27916 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.128847 27916 heartbeater.cc:464] Registering TS with master...
00:25:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.129311 27916 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.128856 27914 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:23.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.131006 23806 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:23.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.131117 23937 ts_manager.cc:194] Re-registered known tserver with Master: a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:23.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.132694 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.130:35595
00:25:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.136143 27914 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:23.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.140985 27257 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:23.241860 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 27038
00:25:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:23.273305 27385 connection.cc:535] server connection from 127.23.2.131:46697 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:25:23.274 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] handling channelInactive
00:25:23.274 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state READY due to: connection closed
00:25:23.274 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:23.275 [DEBUG - Time-limited test] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:23.276 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.277 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.277 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:23.297 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:23.298 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@787619803(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@6f0afe3 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@215ff4b6 -> wakeup thread Time-limited test)) after lookup
00:25:23.298 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:23.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.299 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.316 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:23.317 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:23.317 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [23ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [24ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [24ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [41ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@787619803(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@6f0afe3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@215ff4b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397))) after lookup
00:25:23.318 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:23.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.319 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.356 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:23.356 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:23.357 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [23ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [24ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [24ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [41ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [44ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [44ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@787619803(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@6f0afe3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@215ff4b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407))) after lookup
00:25:23.357 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:23.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.359 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:23.457 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:23.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=182), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [23ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [24ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [24ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [41ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [44ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [44ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [82ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [83ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [84ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@787619803(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@6f0afe3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407) -> (continuation of Deferred@620913184 after retry RPC@1912058200), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@215ff4b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407) -> (continuation of Deferred@620913184 after retry RPC@1912058200))) after lookup
00:25:23.458 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:23.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.459 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.717 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:23.717 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)[L],5635f94622f74554984e41143786b602(127.23.2.129:44273),a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:23.717 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=442), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [23ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [24ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [24ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [41ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [44ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [44ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [82ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [83ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [84ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [183ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [184ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [184ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [438ms] refreshing cache from master, [439ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [441ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@787619803(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@6f0afe3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407) -> (continuation of Deferred@620913184 after retry RPC@1912058200) -> (continuation of Deferred@505560736 after retry RPC@1385378125), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@215ff4b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407) -> (continuation of Deferred@620913184 after retry RPC@1912058200) -> (continuation of Deferred@505560736 after retry RPC@1385378125))) after lookup
00:25:23.718 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] connecting to peer
00:25:23.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] cleaning up while in state CONNECTING due to: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:23.719 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751) for tablet e315584218244a9d9aabe564d716375a: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751
00:25:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.711916 27943 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:25:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.712215 27943 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.713861 27943 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 5635f94622f74554984e41143786b602 (127.23.2.129:44273), 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:24.715410 27727 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:24.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:24.717048 27727 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.720980 27510 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "a24b7a1608504a7eb4e2421899b73f63" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "5635f94622f74554984e41143786b602" is_pre_election: true
00:25:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.721812 27726 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052, 5635f94622f74554984e41143786b602
00:25:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.722257 27943 raft_consensus.cc:2726] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Leader pre-election lost for term 11. Reason: could not achieve majority
00:25:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.743382 27954 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.743587 27954 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.744329 27954 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:24.747074 27385 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.753751 27843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63" is_pre_election: true
00:25:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.754302 27843 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 10.
00:25:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.755401 27385 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.755751 27954 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Leader pre-election won for term 12
00:25:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.755880 27954 raft_consensus.cc:482] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Starting leader election (detected failure of leader 24637a1459374c63935f5fc8cd1b1052)
00:25:24.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.755954 27954 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 11 FOLLOWER]: Advancing to term 12
00:25:24.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.758069 27954 raft_consensus.cc:504] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:24.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.758760 27954 leader_election.cc:288] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 12 election: Requested vote from peers 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751), a24b7a1608504a7eb4e2421899b73f63 (127.23.2.130:42999)
00:25:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.759784 27843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e315584218244a9d9aabe564d716375a" candidate_uuid: "5635f94622f74554984e41143786b602" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "a24b7a1608504a7eb4e2421899b73f63"
00:25:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.760062 27843 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 10 FOLLOWER]: Advancing to term 12
00:25:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:24.760490 27385 leader_election.cc:334] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111)
00:25:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.764410 27843 raft_consensus.cc:2454] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5635f94622f74554984e41143786b602 in term 12.
00:25:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.765019 27385 leader_election.cc:302] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5635f94622f74554984e41143786b602, a24b7a1608504a7eb4e2421899b73f63; no voters: 24637a1459374c63935f5fc8cd1b1052
00:25:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.765400 27954 raft_consensus.cc:2781] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 12 FOLLOWER]: Leader election won for term 12
00:25:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.765892 27954 raft_consensus.cc:686] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [term 12 LEADER]: Becoming Leader. State: Replica: 5635f94622f74554984e41143786b602, State: Running, Role: LEADER
00:25:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.766474 27954 consensus_queue.cc:237] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:24.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:24.770749 23937 catalog_manager.cc:5495] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 reported cstate change: term changed from 11 to 12, leader changed from 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131) to 5635f94622f74554984e41143786b602 (127.23.2.129). New cstate: current_term: 12 leader_uuid: "5635f94622f74554984e41143786b602" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } health_report { overall_health: UNKNOWN } } }
00:25:24.996 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e315584218244a9d9aabe564d716375a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721' with partition [<start>, <end>)
00:25:24.997 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=e315584218244a9d9aabe564d716375a@[24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751),5635f94622f74554984e41143786b602(127.23.2.129:44273)[L],a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)]}]
00:25:24.997 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=e315584218244a9d9aabe564d716375a, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1722), Traces: [0ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [2ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [2ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [23ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [24ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [24ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [41ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [43ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [44ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [44ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [82ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [83ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [84ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [183ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [184ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [184ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [438ms] refreshing cache from master, [439ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [441ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, [443ms] sending RPC to server 24637a1459374c63935f5fc8cd1b1052, [444ms] received response from server 24637a1459374c63935f5fc8cd1b1052: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [444ms] delaying RPC due to: Network error: Failed to connect to peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751): Connection refused: /127.23.2.131:35751, [1718ms] refreshing cache from master, [1718ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.2.188:46541, [1721ms] Sub RPC GetTableLocations: received response from server master-127.23.2.188:46541: OK, deferred=Deferred@787619803(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1830529130@6f0afe3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407) -> (continuation of Deferred@620913184 after retry RPC@1912058200) -> (continuation of Deferred@505560736 after retry RPC@1385378125) -> (continuation of Deferred@2016777971 after retry RPC@478214297), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@215ff4b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1069680156 after retry RPC@732135397) -> (continuation of Deferred@2139874324 after retry RPC@1340600407) -> (continuation of Deferred@620913184 after retry RPC@1912058200) -> (continuation of Deferred@505560736 after retry RPC@1385378125) -> (continuation of Deferred@2016777971 after retry RPC@478214297))) after lookup
00:25:24.998 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] connecting to peer
00:25:24.999 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] Successfully connected to peer
00:25:25.018 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc01e5303, L:/127.0.0.1:46402 - R:/127.23.2.129:44273] using TOKEN/null
00:25:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.024227 27843 raft_consensus.cc:1261] T e315584218244a9d9aabe564d716375a P a24b7a1608504a7eb4e2421899b73f63 [term 12 FOLLOWER]: Refusing update from remote peer 5635f94622f74554984e41143786b602: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 42. (index mismatch)
00:25:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.024593 27385 consensus_peers.cc:487] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 -> Peer 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751): Couldn't send request to peer 24637a1459374c63935f5fc8cd1b1052. Status: Network error: Client connection negotiation failed: client connection to 127.23.2.131:35751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:25:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.025194 27965 consensus_queue.cc:1035] T e315584218244a9d9aabe564d716375a P 5635f94622f74554984e41143786b602 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
00:25:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.035373 27984 mvcc.cc:205] Tried to move back new op lower bound from 7317480755289612288 to 7317480754248880128. Current Snapshot: MvccSnapshot[applied={T|T < 7317480755289612288}]
00:25:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.036309 27986 mvcc.cc:205] Tried to move back new op lower bound from 7317480755289612288 to 7317480754248880128. Current Snapshot: MvccSnapshot[applied={T|T < 7317480755289612288}]
00:25:25.054 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.2.131:35751
00:25:25.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.055951 23562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:25:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.426054 27998 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:25:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.426393 27998 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:25:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.426527 27998 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:25:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.434139 27998 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:25:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.434386 27998 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.131
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.444319 27998 tablet_server_runner.cc:78] Tablet server non-default flags:
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:39221
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:25:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:35751
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46861
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:44663,127.23.2.189:41391,127.23.2.188:46541
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/logs
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 22:51:51 UTC on ccycloud.quasar-sbbrqv.root.comops.site
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581606
00:25:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:25:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.446131 27998 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:25:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.448199 27998 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:25:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.462942 28004 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.462966 28006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:25:25.462904 28009 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:25:25.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.465204 27998 server_base.cc:1034] running on GCE node
00:25:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.465979 27998 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:25:25.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.467157 27998 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:25:25.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.468425 27998 hybrid_clock.cc:648] HybridClock initialized: now 1786494325468338 us; error 60 us; skew 500 ppm
00:25:25.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.468748 27998 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:25:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.471423 27998 webserver.cc:460] Webserver started at http://127.23.2.131:46861/ using document root <none> and password file <none>
00:25:25.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.472232 27998 fs_manager.cc:359] Metadata directory not provided
00:25:25.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.472352 27998 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:25:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.479084 27998 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
00:25:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.482705 28018 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:25:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.484504 27998 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
00:25:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.484691 27998 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data,/tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a1459374c63935f5fc8cd1b1052"
00:25:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:24:54 on dist-test-slave-6bbx"
00:25:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.485759 27998 fs_report.cc:383] FS layout report
00:25:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/wal
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/data
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:25:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:25:25.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.509557 27998 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
00:25:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.510485 27998 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:25:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.510751 27998 kserver.cc:163] Server-wide thread pool size limit: 3276
00:25:25.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.511910 27998 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:25:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.514225 28029 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:25:25.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.520498 27998 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:25:25.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.520633 27998 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:25:25.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.520736 27998 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
00:25:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.524039 27998 ts_tablet_manager.cc:610] Registered 1 tablets
00:25:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.524430 27998 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.004s	sys 0.000s
00:25:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.524609 28029 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap starting.
00:25:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.561126 28029 log.cc:824] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Log is configured to *not* fsync() on all Append() calls
00:25:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.583333 27998 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.131:35751
00:25:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.583344 28210 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.2.131:35751 every 8 connection(s)
00:25:25.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.585191 27998 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLgeSWz/test-tmp/mini-kudu-cluster449937859817997086/ts-2/data/info.pb
00:25:25.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.585546 23562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu as pid 27998
00:25:25.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:25:25.610 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:25.619 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494294721, tablet=e315584218244a9d9aabe564d716375a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:25:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.626005 28029 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:25:25.626 [DEBUG - main] (Connection.java:575) [peer master-127.23.2.190:44663(127.23.2.190:44663)] disconnecting while in state READY
00:25:25.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.626866 28029 tablet_bootstrap.cc:492] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Bootstrap complete.
00:25:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.628434 28029 ts_tablet_manager.cc:1397] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent bootstrapping tablet: real 0.104s	user 0.078s	sys 0.015s
00:25:25.637 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.2.190:44663(127.23.2.190:44663)] handling channelInactive
00:25:25.638 [DEBUG - main] (Connection.java:575) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] disconnecting while in state READY
00:25:25.639 [DEBUG - main] (Connection.java:575) [peer master-127.23.2.189:41391(127.23.2.189:41391)] disconnecting while in state READY
00:25:25.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.2.190:44663(127.23.2.190:44663)] cleaning up while in state READY due to: connection closed
00:25:25.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.637063 28029 raft_consensus.cc:348] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:25.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.640120 28029 raft_consensus.cc:729] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24637a1459374c63935f5fc8cd1b1052, State: Initialized, Role: FOLLOWER
00:25:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.643215 28211 heartbeater.cc:347] Connected to a master server at 127.23.2.188:46541
00:25:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.643641 28211 heartbeater.cc:464] Registering TS with master...
00:25:25.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.644883 28211 heartbeater.cc:511] Master 127.23.2.188:46541 requested a full tablet report, sending...
00:25:25.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.643581 28029 consensus_queue.cc:260] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5635f94622f74554984e41143786b602" member_type: VOTER last_known_addr { host: "127.23.2.129" port: 44273 } } peers { permanent_uuid: "24637a1459374c63935f5fc8cd1b1052" member_type: VOTER last_known_addr { host: "127.23.2.131" port: 35751 } } peers { permanent_uuid: "a24b7a1608504a7eb4e2421899b73f63" member_type: VOTER last_known_addr { host: "127.23.2.130" port: 42999 } }
00:25:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.647818 23937 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.649077 23937 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.2.131:36973
00:25:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.653237 28029 ts_tablet_manager.cc:1428] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052: Time spent starting tablet: real 0.025s	user 0.015s	sys 0.000s
00:25:25.654 [DEBUG - main] (Connection.java:575) [peer 24637a1459374c63935f5fc8cd1b1052(127.23.2.131:35751)] disconnecting while in state TERMINATED
00:25:25.654 [DEBUG - main] (Connection.java:575) [peer master-127.23.2.188:46541(127.23.2.188:46541)] disconnecting while in state READY
00:25:25.654 [DEBUG - main] (Connection.java:575) [peer a24b7a1608504a7eb4e2421899b73f63(127.23.2.130:42999)] disconnecting while in state TERMINATED
00:25:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.655449 28211 heartbeater.cc:503] Master 127.23.2.188:46541 was elected leader, sending a full tablet report...
00:25:25.656 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] handling channelInactive
00:25:25.656 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.2.188:46541(127.23.2.188:46541)] handling channelInactive
00:25:25.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.2.188:46541(127.23.2.188:46541)] cleaning up while in state READY due to: connection closed
00:25:25.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5635f94622f74554984e41143786b602(127.23.2.129:44273)] cleaning up while in state READY due to: connection closed
00:25:25.657 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:25:25.661 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.2.189:41391(127.23.2.189:41391)] handling channelInactive
00:25:25.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.2.189:41391(127.23.2.189:41391)] cleaning up while in state READY due to: connection closed
00:25:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.662717 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 27358
00:25:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.671648 28212 heartbeater.cc:347] Connected to a master server at 127.23.2.190:44663
00:25:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.673600 28212 heartbeater.cc:464] Registering TS with master...
00:25:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.674305 28212 heartbeater.cc:511] Master 127.23.2.190:44663 requested a full tablet report, sending...
00:25:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.676355 23653 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.680393 28227 heartbeater.cc:347] Connected to a master server at 127.23.2.189:41391
00:25:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.680572 28227 heartbeater.cc:464] Registering TS with master...
00:25:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.680953 28227 heartbeater.cc:511] Master 127.23.2.189:41391 requested a full tablet report, sending...
00:25:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.682581 23806 ts_manager.cc:194] Re-registered known tserver with Master: 24637a1459374c63935f5fc8cd1b1052 (127.23.2.131:35751)
00:25:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.686090 27591 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.686196 28125 raft_consensus.cc:3037] T e315584218244a9d9aabe564d716375a P 24637a1459374c63935f5fc8cd1b1052 [term 11 FOLLOWER]: Advancing to term 12
00:25:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.797148 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 27358
00:25:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.831768 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 27697
00:25:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.839244 27907 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:25.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.932284 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 27697
00:25:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.959787 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 27998
00:25:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:25.967010 28204 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.061545 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 27998
00:25:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.087926 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 23586
00:25:26.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.089159 23713 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.208135 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 23586
00:25:26.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.240283 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 23728
00:25:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.241495 23858 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.357939 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 23728
00:25:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.387852 23562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu pid 23870
00:25:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.388979 23999 generic_service.cc:196] Checking for leaks (request via RPC)
00:25:26.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:25:26.508689 23562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLgeSWz/build/asan/bin/kudu with pid 23870
00:25:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:25:26Z chronyd exiting

Time: 37.365

OK (1 test)

