JUnit version 4.13.2
.12:09:02.577 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783512542566
12:09:02.643 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
12:09:02.684 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
12:09:02.697 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkxu2RH/test-tmp
12:09:02.723 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
12:09:02.726 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkxu2RH/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
12:09:02.977 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

12:09:03.166 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:09:03.168 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

12:09:03.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.171629 29807 mini_oidc.cc:150] Starting JWKS server
12:09:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.174702 29807 webserver.cc:533] Webserver started at http://127.0.0.1:36621/ using document root <none> and password file <none>
12:09:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.174944 29807 mini_oidc.cc:186] Starting OIDC Discovery server
12:09:03.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.175601 29807 webserver.cc:533] Webserver started at http://127.0.0.1:39381/ using document root <none> and password file <none>
12:09:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:09:03Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
12:09:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:09:03Z Disabled control of system clock
12:09:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.206713 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal
12:09:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data
12:09:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/logs
12:09:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data/info.pb
12:09:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:09:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:09:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:09:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.254:34347
12:09:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.254
12:09:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:09:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:09:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36621/jwks.json with env {}
12:09:03.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.577911 29818 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:03.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.578370 29818 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:03.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.578465 29818 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.591092 29818 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:09:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.591205 29818 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.591264 29818 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:09:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.591310 29818 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:09:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.591367 29818 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:09:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.591442 29818 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36621/jwks.json
12:09:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.603884 29818 master_runner.cc:387] Master server non-default flags:
12:09:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.254:34347
12:09:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36621/jwks.json
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data/info.pb
12:09:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.254
12:09:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/logs
12:09:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:09:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.606181 29818 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.608619 29818 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.619797 29824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:03.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.619846 29823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.621932 29826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.623353 29818 server_base.cc:1061] running on GCE node
12:09:03.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.624567 29818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.626390 29818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.627779 29818 hybrid_clock.cc:648] HybridClock initialized: now 1783512543627629 us; error 133 us; skew 500 ppm
12:09:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.628353 29818 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:03.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.631134 29818 webserver.cc:533] Webserver started at http://127.29.27.254:40269/ using document root <none> and password file <none>
12:09:03.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.632025 29818 fs_manager.cc:362] Metadata directory not provided
12:09:03.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.632140 29818 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:03.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.632562 29818 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:09:03.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.636377 29818 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data/instance:
12:09:03.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6488a2af92343f28c0f99af0e67169d"
12:09:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:03 on dist-test-slave-nb2j"
12:09:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.637535 29818 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal/instance:
12:09:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6488a2af92343f28c0f99af0e67169d"
12:09:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:03 on dist-test-slave-nb2j"
12:09:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.645105 29818 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.004s
12:09:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.649538 29832 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.651438 29818 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
12:09:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.651634 29818 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal
12:09:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6488a2af92343f28c0f99af0e67169d"
12:09:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:03 on dist-test-slave-nb2j"
12:09:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.651959 29818 fs_report.cc:389] FS layout report
12:09:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal
12:09:03.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal
12:09:03.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data/data
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:03.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:03.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:03.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:03.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.688162 29818 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.689129 29818 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.689512 29818 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.714038 29818 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.254:34347
12:09:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.714057 29883 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.254:34347 every 8 connection(s)
12:09:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.716218 29818 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/data/info.pb
12:09:03.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.719609 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 29818
12:09:03.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.720036 29807 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-0/wal/instance
12:09:03.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.721496 29884 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:09:03.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.723402 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal
12:09:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data
12:09:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/logs
12:09:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data/info.pb
12:09:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:09:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:09:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:09:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.253:43153
12:09:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.253
12:09:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:09:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:09:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36621/jwks.json with env {}
12:09:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.732197 29884 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } has no permanent_uuid. Determining permanent_uuid...
12:09:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.757006 29884 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } has no permanent_uuid. Determining permanent_uuid...
12:09:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.758801 29836 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.253:43153: connect: Connection refused (error 111)
12:09:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.761624 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.253:43153: Network error: Client connection negotiation failed: client connection to 127.29.27.253:43153: connect: Connection refused (error 111)
12:09:03.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.811141 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } attempt: 1
12:09:03.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.813678 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.253:43153: Network error: Client connection negotiation failed: client connection to 127.29.27.253:43153: connect: Connection refused (error 111)
12:09:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.881990 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } attempt: 2
12:09:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.884446 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.253:43153: Network error: Client connection negotiation failed: client connection to 127.29.27.253:43153: connect: Connection refused (error 111)
12:09:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:03.975817 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } attempt: 3
12:09:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:03.978848 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.253:43153: Network error: Client connection negotiation failed: client connection to 127.29.27.253:43153: connect: Connection refused (error 111)
12:09:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.114343 29886 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.114809 29886 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.114941 29886 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.122192 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } attempt: 4
12:09:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.124559 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.253:43153: Network error: Client connection negotiation failed: client connection to 127.29.27.253:43153: connect: Connection refused (error 111)
12:09:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.124936 29886 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:09:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.125003 29886 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.125057 29886 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:09:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.125097 29886 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:09:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.125152 29886 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:09:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.125187 29886 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36621/jwks.json
12:09:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.136775 29886 master_runner.cc:387] Master server non-default flags:
12:09:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data
12:09:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal
12:09:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:09:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.253:43153
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36621/jwks.json
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data/info.pb
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.253
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/logs
12:09:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:09:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.139178 29886 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:04.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.142014 29886 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.151712 29895 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.151736 29894 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.153278 29897 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.155521 29886 server_base.cc:1061] running on GCE node
12:09:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.156538 29886 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.158478 29886 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:04.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.159786 29886 hybrid_clock.cc:648] HybridClock initialized: now 1783512544159709 us; error 74 us; skew 500 ppm
12:09:04.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.160293 29886 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:04.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.162925 29886 webserver.cc:533] Webserver started at http://127.29.27.253:34261/ using document root <none> and password file <none>
12:09:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.163928 29886 fs_manager.cc:362] Metadata directory not provided
12:09:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.164083 29886 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.164587 29886 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:09:04.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.167290 29886 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data/instance:
12:09:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f94d89fdb6142af9161025a4174696d"
12:09:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:04 on dist-test-slave-nb2j"
12:09:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.168329 29886 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal/instance:
12:09:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f94d89fdb6142af9161025a4174696d"
12:09:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:04 on dist-test-slave-nb2j"
12:09:04.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.174731 29886 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
12:09:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.178974 29903 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.180642 29886 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
12:09:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.180861 29886 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal
12:09:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f94d89fdb6142af9161025a4174696d"
12:09:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:04 on dist-test-slave-nb2j"
12:09:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.181128 29886 fs_report.cc:389] FS layout report
12:09:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal
12:09:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data/data
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:04.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:04.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:04.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:04.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.203320 29886 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.204475 29886 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.204806 29886 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:04.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.226766 29886 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.253:43153
12:09:04.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.226804 29954 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.253:43153 every 8 connection(s)
12:09:04.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.228628 29886 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/data/info.pb
12:09:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.233330 29955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:09:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.237634 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 29886
12:09:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.237918 29807 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-1/wal/instance
12:09:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.239135 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal
12:09:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/logs
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data/info.pb
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.252:40279
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.252
12:09:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:09:04.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36621/jwks.json with env {}
12:09:04.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.240548 29955 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.257545 29955 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.268000 29955 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.269315 29906 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.271035 29955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.320420 29955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 1
12:09:04.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.323009 29955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.391291 29955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 2
12:09:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.393726 29955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.424039 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } attempt: 5
12:09:04.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.431346 29884 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.433641 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.485039 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 1
12:09:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.485630 29955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 3
12:09:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.488759 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.489641 29955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.557001 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 2
12:09:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.559551 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.621124 29957 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.621488 29957 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.621573 29957 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.630328 29957 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:09:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.630436 29957 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.630491 29957 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:09:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.630532 29957 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:09:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.630584 29957 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:09:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.630618 29957 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36621/jwks.json
12:09:04.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.632907 29955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 4
12:09:04.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.635222 29955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.645792 29957 master_runner.cc:387] Master server non-default flags:
12:09:04.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:04.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:04.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:04.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:09:04.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.252:40279
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36621/jwks.json
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data/info.pb
12:09:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.252
12:09:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/logs
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.647542 29957 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.649621 29957 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:04.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.658268 29966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:04.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.658533 29965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:04.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.659090 29968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.660539 29957 server_base.cc:1061] running on GCE node
12:09:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.661525 29957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.663192 29957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:04.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.664474 29957 hybrid_clock.cc:648] HybridClock initialized: now 1783512544664421 us; error 40 us; skew 500 ppm
12:09:04.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.664965 29957 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.665863 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 3
12:09:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.668062 29957 webserver.cc:533] Webserver started at http://127.29.27.252:40769/ using document root <none> and password file <none>
12:09:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.668469 29884 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.27.252:40279: Network error: Client connection negotiation failed: client connection to 127.29.27.252:40279: connect: Connection refused (error 111)
12:09:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.669040 29957 fs_manager.cc:362] Metadata directory not provided
12:09:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.669152 29957 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.669453 29957 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.672070 29957 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data/instance:
12:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912b29e54fe0440bbb6f4206f6b2d3cb"
12:09:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:04 on dist-test-slave-nb2j"
12:09:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.672889 29957 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal/instance:
12:09:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912b29e54fe0440bbb6f4206f6b2d3cb"
12:09:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:04 on dist-test-slave-nb2j"
12:09:04.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.679141 29957 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.003s	sys 0.003s
12:09:04.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.683049 29974 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.684754 29957 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.684916 29957 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912b29e54fe0440bbb6f4206f6b2d3cb"
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:04 on dist-test-slave-nb2j"
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.685117 29957 fs_report.cc:389] FS layout report
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data/data
12:09:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:04.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.706284 29957 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:04.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.707432 29957 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.707823 29957 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.731302 29957 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.252:40279
12:09:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.731339 30025 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.252:40279 every 8 connection(s)
12:09:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.732955 29957 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/data/info.pb
12:09:04.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.737686 30026 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:09:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.740988 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 29957
12:09:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.741199 29807 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/master-2/wal/instance
12:09:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.745854 30026 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.776827 30026 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.784575 30026 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } has no permanent_uuid. Determining permanent_uuid...
12:09:04.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.797230 30026 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: Bootstrap starting.
12:09:04.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.801080 30026 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: Neither blocks nor log segments found. Creating new log.
12:09:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.802382 30026 log.cc:826] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: Log is configured to *not* fsync() on all Append() calls
12:09:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.806067 30026 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: No bootstrap required, opened a new log
12:09:04.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.812163 30026 raft_consensus.cc:359] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.812553 30026 raft_consensus.cc:385] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:09:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.812683 30026 raft_consensus.cc:740] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 912b29e54fe0440bbb6f4206f6b2d3cb, State: Initialized, Role: FOLLOWER
12:09:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.813549 30026 consensus_queue.cc:260] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.814642 30033 sys_catalog.cc:455] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:04.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.814919 30033 sys_catalog.cc:458] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: This master's current role is: FOLLOWER
12:09:04.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.816035 30026 sys_catalog.cc:565] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: configured and running, proceeding with master startup.
12:09:04.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.827512 30044 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:09:04.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.828235 30044 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:09:04.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.824488 29957 master_runner.cc:187] Error getting master registration for 127.29.27.254:34347: OK, instance_id { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" instance_seqno: 1783512543707072 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:09:04.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.830911 29957 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:09:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.845717 29884 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 4
12:09:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.859903 29884 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: Bootstrap starting.
12:09:04.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.865305 29884 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: Neither blocks nor log segments found. Creating new log.
12:09:04.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.866864 29884 log.cc:826] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: Log is configured to *not* fsync() on all Append() calls
12:09:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.871450 29884 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: No bootstrap required, opened a new log
12:09:04.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.879930 29884 raft_consensus.cc:359] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.880339 29884 raft_consensus.cc:385] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:09:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.880474 29884 raft_consensus.cc:740] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6488a2af92343f28c0f99af0e67169d, State: Initialized, Role: FOLLOWER
12:09:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.881286 29884 consensus_queue.cc:260] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.882761 30047 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:04.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.883389 30047 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: This master's current role is: FOLLOWER
12:09:04.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.884147 29884 sys_catalog.cc:565] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: configured and running, proceeding with master startup.
12:09:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.895191 30058 catalog_manager.cc:1594] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:09:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.895972 30058 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:09:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.894559 29818 master_runner.cc:187] Error getting master registration for 127.29.27.253:43153: OK, instance_id { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" instance_seqno: 1783512544220972 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:09:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.902082 29818 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:09:04.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.934527 29955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } attempt: 5
12:09:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.943178 29838 tablet.cc:2406] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:09:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.948733 29955 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: Bootstrap starting.
12:09:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.952796 30033 raft_consensus.cc:493] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:09:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.953105 30033 raft_consensus.cc:515] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.953343 29955 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: Neither blocks nor log segments found. Creating new log.
12:09:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.954829 29955 log.cc:826] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: Log is configured to *not* fsync() on all Append() calls
12:09:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.954978 30033 leader_election.cc:290] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6488a2af92343f28c0f99af0e67169d (127.29.27.254:34347), 7f94d89fdb6142af9161025a4174696d (127.29.27.253:43153)
12:09:04.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.956851 29859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6488a2af92343f28c0f99af0e67169d" is_pre_election: true
12:09:04.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.957459 29859 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 912b29e54fe0440bbb6f4206f6b2d3cb in term 0.
12:09:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.958772 29977 leader_election.cc:304] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 912b29e54fe0440bbb6f4206f6b2d3cb, c6488a2af92343f28c0f99af0e67169d; no voters: 
12:09:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.959298 30033 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Leader pre-election won for term 1
12:09:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.959473 30033 raft_consensus.cc:493] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:09:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.959687 30033 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 0 FOLLOWER]: Advancing to term 1
12:09:04.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:04.960258 29980 tablet.cc:2406] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:09:04.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.963064 30033 raft_consensus.cc:515] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.965382 29859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6488a2af92343f28c0f99af0e67169d"
12:09:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.965684 29859 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 0 FOLLOWER]: Advancing to term 1
12:09:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.966398 29955 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: No bootstrap required, opened a new log
12:09:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.969131 29859 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 912b29e54fe0440bbb6f4206f6b2d3cb in term 1.
12:09:04.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.969717 29977 leader_election.cc:304] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 912b29e54fe0440bbb6f4206f6b2d3cb, c6488a2af92343f28c0f99af0e67169d; no voters: 
12:09:04.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.973299 30033 leader_election.cc:290] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [CANDIDATE]: Term 1 election: Requested vote from peers c6488a2af92343f28c0f99af0e67169d (127.29.27.254:34347), 7f94d89fdb6142af9161025a4174696d (127.29.27.253:43153)
12:09:04.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.974706 29955 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.975625 29955 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:09:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.975785 30033 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 1 FOLLOWER]: Leader election won for term 1
12:09:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.976105 29955 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f94d89fdb6142af9161025a4174696d, State: Initialized, Role: FOLLOWER
12:09:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.977027 29955 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.977296 30033 raft_consensus.cc:697] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [term 1 LEADER]: Becoming Leader. State: Replica: 912b29e54fe0440bbb6f4206f6b2d3cb, State: Running, Role: LEADER
12:09:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.979976 30033 consensus_queue.cc:237] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } }
12:09:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.993196 29955 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: configured and running, proceeding with master startup.
12:09:04.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.993242 29929 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f94d89fdb6142af9161025a4174696d"
12:09:04.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.993739 29929 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [term 0 FOLLOWER]: Advancing to term 1
12:09:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.995889 29930 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f94d89fdb6142af9161025a4174696d" is_pre_election: true
12:09:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.999292 29929 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 912b29e54fe0440bbb6f4206f6b2d3cb in term 1.
12:09:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.992527 30062 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:04.999820 30062 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: This master's current role is: FOLLOWER
12:09:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.006711 30061 sys_catalog.cc:455] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 912b29e54fe0440bbb6f4206f6b2d3cb. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.007680 30061 sys_catalog.cc:458] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: This master's current role is: LEADER
12:09:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.028352 30066 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
12:09:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.031426 30066 catalog_manager.cc:1520] Initializing Kudu cluster ID...
12:09:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.045555 29929 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [term 1 FOLLOWER]: Refusing update from remote peer 912b29e54fe0440bbb6f4206f6b2d3cb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:09:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.048062 30061 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
12:09:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.050351 29859 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [term 1 FOLLOWER]: Refusing update from remote peer 912b29e54fe0440bbb6f4206f6b2d3cb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:09:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.051833 30033 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:09:05.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.075865 30047 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 912b29e54fe0440bbb6f4206f6b2d3cb. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.076179 30047 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: This master's current role is: FOLLOWER
12:09:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.081802 30071 mvcc.cc:204] Tried to move back new op lower bound from 7305267384491188224 to 7305267384274882560. Current Snapshot: MvccSnapshot[applied={T|T < 7305267384491188224}]
12:09:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.083377 30062 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 912b29e54fe0440bbb6f4206f6b2d3cb. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.083642 30062 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: This master's current role is: FOLLOWER
12:09:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.088581 30047 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.088855 30047 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d [sys.catalog]: This master's current role is: FOLLOWER
12:09:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.092698 30077 mvcc.cc:204] Tried to move back new op lower bound from 7305267384491188224 to 7305267384274882560. Current Snapshot: MvccSnapshot[applied={T|T < 7305267384491188224 or (T in {7305267384491188224})}]
12:09:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.093669 30081 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
12:09:05.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.097374 30061 sys_catalog.cc:455] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.097668 30061 sys_catalog.cc:458] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: This master's current role is: LEADER
12:09:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.101246 30061 sys_catalog.cc:455] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.101521 30061 sys_catalog.cc:458] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb [sys.catalog]: This master's current role is: LEADER
12:09:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.101557 30066 catalog_manager.cc:1383] Generated new cluster ID: 183246b52b484a158b520d7eb42c3fcf
12:09:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.101668 30066 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
12:09:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.102968 30062 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6488a2af92343f28c0f99af0e67169d" member_type: VOTER last_known_addr { host: "127.29.27.254" port: 34347 } } peers { permanent_uuid: "7f94d89fdb6142af9161025a4174696d" member_type: VOTER last_known_addr { host: "127.29.27.253" port: 43153 } } peers { permanent_uuid: "912b29e54fe0440bbb6f4206f6b2d3cb" member_type: VOTER last_known_addr { host: "127.29.27.252" port: 40279 } } }
12:09:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.103194 30062 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d [sys.catalog]: This master's current role is: FOLLOWER
12:09:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.123128 30066 catalog_manager.cc:1406] Generated new certificate authority record
12:09:05.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.125475 30066 catalog_manager.cc:1540] Loading token signing keys...
12:09:05.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.151572 30066 catalog_manager.cc:6070] T 00000000000000000000000000000000 P 912b29e54fe0440bbb6f4206f6b2d3cb: Generated new TSK 0
12:09:05.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.152908 30066 catalog_manager.cc:1550] Initializing in-progress tserver states...
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.187556 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:0
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.631435 30086 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.631881 30086 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.631973 30086 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:05.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.640776 30086 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:05.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.640954 30086 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.193
12:09:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.651508 30086 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:0
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.653620 30086 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.655827 30086 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:05.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.666163 30091 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:05.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.667351 30092 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:05.674830 30094 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.676162 30086 server_base.cc:1061] running on GCE node
12:09:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.676999 30086 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.678200 30086 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.679440 30086 hybrid_clock.cc:648] HybridClock initialized: now 1783512545679382 us; error 53 us; skew 500 ppm
12:09:05.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.679886 30086 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:05.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.682444 30086 webserver.cc:533] Webserver started at http://127.29.27.193:44091/ using document root <none> and password file <none>
12:09:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.683311 30086 fs_manager.cc:362] Metadata directory not provided
12:09:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.683449 30086 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.683972 30086 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:09:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.686612 30086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/instance:
12:09:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:05 on dist-test-slave-nb2j"
12:09:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.687569 30086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal/instance:
12:09:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:05.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:05 on dist-test-slave-nb2j"
12:09:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.693921 30086 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
12:09:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.698091 30100 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.700099 30086 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
12:09:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.700335 30086 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:05 on dist-test-slave-nb2j"
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.700636 30086 fs_report.cc:389] FS layout report
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/data
12:09:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.732585 30086 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.733727 30086 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.734109 30086 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.735325 30086 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:05.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.737484 30086 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:09:05.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.737636 30086 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:09:05.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.737773 30086 ts_tablet_manager.cc:616] Registered 0 tablets
12:09:05.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.737874 30086 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:09:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.777817 30086 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.193:44909
12:09:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.777915 30212 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.193:44909 every 8 connection(s)
12:09:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.779480 30086 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.787210 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 30086
12:09:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.787446 29807 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal/instance
12:09:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.793380 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:0
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:05.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.800567 30215 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.801064 30215 heartbeater.cc:461] Registering TS with master...
12:09:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.801549 30214 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.801716 30214 heartbeater.cc:461] Registering TS with master...
12:09:05.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.802716 30214 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:05.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.802793 30215 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.805747 29919 ts_manager.cc:194] Registered new tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.805851 29849 ts_manager.cc:194] Registered new tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.809912 30213 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.810111 30213 heartbeater.cc:461] Registering TS with master...
12:09:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.810755 30213 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.812915 29991 ts_manager.cc:194] Registered new tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:05.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.815459 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.193:40307
12:09:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.898342 30058 catalog_manager.cc:1295] Loaded cluster ID: 183246b52b484a158b520d7eb42c3fcf
12:09:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.898494 30058 catalog_manager.cc:1588] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: loading cluster ID for follower catalog manager: success
12:09:05.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.901772 30058 catalog_manager.cc:1610] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: acquiring CA information for follower catalog manager: success
12:09:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:05.904090 30058 catalog_manager.cc:1638] T 00000000000000000000000000000000 P c6488a2af92343f28c0f99af0e67169d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:09:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.099864 30081 catalog_manager.cc:1295] Loaded cluster ID: 183246b52b484a158b520d7eb42c3fcf
12:09:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.100029 30081 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: loading cluster ID for follower catalog manager: success
12:09:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.104835 30081 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: acquiring CA information for follower catalog manager: success
12:09:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.107915 30081 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 7f94d89fdb6142af9161025a4174696d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:09:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.169339 30223 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.169728 30223 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.169849 30223 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:06.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.179493 30223 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.179756 30223 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.194
12:09:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.191102 30223 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:0
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.193475 30223 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.195498 30223 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:06.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.205940 30229 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.207178 30230 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:06.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.209612 30232 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.212253 30223 server_base.cc:1061] running on GCE node
12:09:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.213179 30223 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.214636 30223 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.216190 30223 hybrid_clock.cc:648] HybridClock initialized: now 1783512546216071 us; error 98 us; skew 500 ppm
12:09:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.216616 30223 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:06.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.219637 30223 webserver.cc:533] Webserver started at http://127.29.27.194:39915/ using document root <none> and password file <none>
12:09:06.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.220503 30223 fs_manager.cc:362] Metadata directory not provided
12:09:06.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.220595 30223 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:06.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.221029 30223 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:09:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.223784 30223 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/instance:
12:09:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.224565 30223 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal/instance:
12:09:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:06.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.231508 30223 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.003s
12:09:06.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.236167 30238 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.238055 30223 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.238250 30223 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.238479 30223 fs_report.cc:389] FS layout report
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/data
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:06.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.264304 30223 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:06.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.265493 30223 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:06.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.265756 30223 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:06.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.266885 30223 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.268824 30223 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:09:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.268955 30223 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:09:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.269059 30223 ts_tablet_manager.cc:616] Registered 0 tablets
12:09:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.269160 30223 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:09:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.316093 30223 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.194:36885
12:09:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.316105 30350 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.194:36885 every 8 connection(s)
12:09:06.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.317716 30223 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.322839 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 30223
12:09:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.323038 29807 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal/instance
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.331040 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:0
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:06.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:06.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:06.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.355679 30353 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.356096 30353 heartbeater.cc:461] Registering TS with master...
12:09:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.357287 30353 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.359373 29919 ts_manager.cc:194] Registered new tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.366020 30352 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.366225 30352 heartbeater.cc:461] Registering TS with master...
12:09:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.366793 30352 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.371043 29849 ts_manager.cc:194] Registered new tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.372560 30351 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.372723 30351 heartbeater.cc:461] Registering TS with master...
12:09:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.373234 30351 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:06.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.374294 29991 ts_manager.cc:194] Registered new tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.375316 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.194:51729
12:09:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.729031 30359 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.729367 30359 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.729451 30359 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.738615 30359 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.738806 30359 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.195
12:09:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.749512 30359 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:0
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.751266 30359 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.753257 30359 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.764012 30366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.764133 30367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:06.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:06.765228 30369 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:06.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.766108 30359 server_base.cc:1061] running on GCE node
12:09:06.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.766776 30359 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.767948 30359 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:06.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.769219 30359 hybrid_clock.cc:648] HybridClock initialized: now 1783512546769143 us; error 66 us; skew 500 ppm
12:09:06.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.769610 30359 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.772257 30359 webserver.cc:533] Webserver started at http://127.29.27.195:35881/ using document root <none> and password file <none>
12:09:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.773109 30359 fs_manager.cc:362] Metadata directory not provided
12:09:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.773252 30359 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.773654 30359 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:09:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.776268 30359 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/instance:
12:09:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.777205 30359 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal/instance:
12:09:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:06.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.783222 30359 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.000s
12:09:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.786902 30375 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.788759 30359 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.788980 30359 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.789244 30359 fs_report.cc:389] FS layout report
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/data
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.818619 30213 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.825284 30359 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.826334 30359 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.826710 30359 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.827916 30359 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.829842 30359 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:09:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.829970 30359 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:09:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.830076 30359 ts_tablet_manager.cc:616] Registered 0 tablets
12:09:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.830161 30359 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:09:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.868108 30359 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.195:41661
12:09:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.868173 30487 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.195:41661 every 8 connection(s)
12:09:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.869727 30359 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.872510 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 30359
12:09:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.872684 29807 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal/instance
12:09:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.895334 30489 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.895524 30488 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.895548 30490 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.895963 30489 heartbeater.cc:461] Registering TS with master...
12:09:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.895959 30488 heartbeater.cc:461] Registering TS with master...
12:09:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.895959 30490 heartbeater.cc:461] Registering TS with master...
12:09:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.897274 30488 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.897310 30490 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.897333 30489 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.899133 29919 ts_manager.cc:194] Registered new tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.899159 29991 ts_manager.cc:194] Registered new tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.899257 29849 ts_manager.cc:194] Registered new tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.900309 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.195:56271
12:09:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:06.907224 29807 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
12:09:06.909 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:09:06.911 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

12:09:06.921 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.29.27.254"
      port: 34347
    }
    bound_http_address {
      host: "127.29.27.254"
      port: 40269
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.29.27.253"
      port: 43153
    }
    bound_http_address {
      host: "127.29.27.253"
      port: 34261
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.29.27.252"
      port: 40279
    }
    bound_http_address {
      host: "127.29.27.252"
      port: 40769
    }
  }
}

12:09:06.925 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

12:09:06.927 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.29.27.193"
      port: 44909
    }
    bound_http_address {
      host: "127.29.27.193"
      port: 44091
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.29.27.194"
      port: 36885
    }
    bound_http_address {
      host: "127.29.27.194"
      port: 39915
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.29.27.195"
      port: 41661
    }
    bound_http_address {
      host: "127.29.27.195"
      port: 35881
    }
  }
}

12:09:06.928 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
12:09:07.149 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
12:09:07.219 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
12:09:07.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:07.377682 30351 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:07.406 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.27.252:40279(127.29.27.252:40279)] connecting to peer
12:09:07.469 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.27.254:34347(127.29.27.254:34347)] connecting to peer
12:09:07.471 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.27.253:43153(127.29.27.253:43153)] connecting to peer
12:09:07.491 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.27.252:40279(127.29.27.252:40279)] Successfully connected to peer
12:09:07.491 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.27.254:34347(127.29.27.254:34347)] Successfully connected to peer
12:09:07.491 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.29.27.253:43153(127.29.27.253:43153)] Successfully connected to peer
12:09:07.597 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.27.252
12:09:07.597 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.27.253
12:09:07.602 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.27.254
12:09:07.833 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8a197be4, L:/127.0.0.1:35858 - R:/127.29.27.252:40279] using SASL/PLAIN
12:09:07.833 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xa4de70d5, L:/127.0.0.1:43900 - R:/127.29.27.253:43153] using SASL/PLAIN
12:09:07.833 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x58dc8c63, L:/127.0.0.1:48278 - R:/127.29.27.254:34347] using SASL/PLAIN
12:09:07.874 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
12:09:07.876 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=511), Traces: [0ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.27.252:40279, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.27.254:34347, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.27.253:43153, [479ms] Sub RPC ConnectToMaster: received response from server master-127.29.27.254:34347: OK, [479ms] Sub RPC ConnectToMaster: received response from server master-127.29.27.253:43153: OK, [479ms] Sub RPC ConnectToMaster: received response from server master-127.29.27.252:40279: OK, deferred=null) after lookup
12:09:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:07.902570 30488 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:07.931425 29990 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35858:
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348"
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:09:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:09:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:09:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:07.933840 29990 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
12:09:07.984 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 34fb04b931b440a8962edf70fef12536
12:09:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.024782 30148 tablet_service.cc:1511] Processing CreateTablet for tablet a061d38df9604855aecaa770f9a58714 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348 [id=34fb04b931b440a8962edf70fef12536]), partition=RANGE (key) PARTITION UNBOUNDED
12:09:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.027127 30148 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a061d38df9604855aecaa770f9a58714. 1 dirs total, 0 dirs full, 0 dirs failed
12:09:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.033174 30423 tablet_service.cc:1511] Processing CreateTablet for tablet a061d38df9604855aecaa770f9a58714 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348 [id=34fb04b931b440a8962edf70fef12536]), partition=RANGE (key) PARTITION UNBOUNDED
12:09:08.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.034828 30423 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a061d38df9604855aecaa770f9a58714. 1 dirs total, 0 dirs full, 0 dirs failed
12:09:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.036104 30286 tablet_service.cc:1511] Processing CreateTablet for tablet a061d38df9604855aecaa770f9a58714 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348 [id=34fb04b931b440a8962edf70fef12536]), partition=RANGE (key) PARTITION UNBOUNDED
12:09:08.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.037643 30286 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a061d38df9604855aecaa770f9a58714. 1 dirs total, 0 dirs full, 0 dirs failed
12:09:08.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.044845 30523 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap starting.
12:09:08.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.050518 30523 tablet_bootstrap.cc:654] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Neither blocks nor log segments found. Creating new log.
12:09:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.052459 30524 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap starting.
12:09:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.053071 30523 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Log is configured to *not* fsync() on all Append() calls
12:09:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.054286 30525 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap starting.
12:09:08.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.056527 30524 tablet_bootstrap.cc:654] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Neither blocks nor log segments found. Creating new log.
12:09:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.057742 30523 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: No bootstrap required, opened a new log
12:09:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.058101 30523 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent bootstrapping tablet: real 0.014s	user 0.009s	sys 0.000s
12:09:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.058224 30524 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Log is configured to *not* fsync() on all Append() calls
12:09:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.058782 30525 tablet_bootstrap.cc:654] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Neither blocks nor log segments found. Creating new log.
12:09:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.060417 30525 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Log is configured to *not* fsync() on all Append() calls
12:09:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.061489 30524 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: No bootstrap required, opened a new log
12:09:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.061794 30524 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent bootstrapping tablet: real 0.010s	user 0.007s	sys 0.000s
12:09:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.063373 30525 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: No bootstrap required, opened a new log
12:09:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.063784 30525 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent bootstrapping tablet: real 0.010s	user 0.009s	sys 0.000s
12:09:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.066251 30523 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.066985 30523 raft_consensus.cc:385] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:09:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.067234 30523 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Initialized, Role: FOLLOWER
12:09:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.069093 30524 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.070084 30524 raft_consensus.cc:385] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:09:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.070209 30524 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4730dea995f64013a3936f9529b63580, State: Initialized, Role: FOLLOWER
12:09:08.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.070940 30524 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.072423 30525 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.073071 30525 raft_consensus.cc:385] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:09:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.073354 30525 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Initialized, Role: FOLLOWER
12:09:08.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.074373 30525 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.080044 30525 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent starting tablet: real 0.016s	user 0.010s	sys 0.004s
12:09:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.081079 30523 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.082340 30524 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent starting tablet: real 0.020s	user 0.016s	sys 0.004s
12:09:08.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:08.084070 30354 tablet.cc:2406] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:09:08.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.092595 30523 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent starting tablet: real 0.034s	user 0.009s	sys 0.009s
12:09:08.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:08.124464 30491 tablet.cc:2406] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:09:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:08.286098 30216 tablet.cc:2406] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:09:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.347592 30529 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:09:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.347908 30529 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.350327 30529 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.358193 30306 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef" is_pre_election: true
12:09:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.358183 30168 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12" is_pre_election: true
12:09:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.358793 30168 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 0.
12:09:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.358980 30306 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 0.
12:09:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.360558 30379 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 
12:09:08.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.361119 30529 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Leader pre-election won for term 1
12:09:08.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.361302 30529 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:09:08.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.361426 30529 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 0 FOLLOWER]: Advancing to term 1
12:09:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.363476 30529 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.364214 30529 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 1 election: Requested vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:08.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.364931 30306 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:08.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.365002 30168 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:08.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.365198 30168 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 0 FOLLOWER]: Advancing to term 1
12:09:08.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.365170 30306 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 0 FOLLOWER]: Advancing to term 1
12:09:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.367431 30306 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 1.
12:09:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.367487 30168 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 1.
12:09:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.368067 30379 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 
12:09:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.368574 30529 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 FOLLOWER]: Leader election won for term 1
12:09:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.369242 30529 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 LEADER]: Becoming Leader. State: Replica: 4730dea995f64013a3936f9529b63580, State: Running, Role: LEADER
12:09:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.369940 30529 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.376183 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4730dea995f64013a3936f9529b63580 (127.29.27.195). New cstate: current_term: 1 leader_uuid: "4730dea995f64013a3936f9529b63580" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:08.412 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 34fb04b931b440a8962edf70fef12536
12:09:08.429 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:08.435 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:08.436 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=null) after lookup
12:09:08.445 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:08.448 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] Successfully connected to peer
12:09:08.529 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x41c857a5, L:/127.0.0.1:50104 - R:/127.29.27.195:41661] using TOKEN/null
12:09:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.540757 30306 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Refusing update from remote peer 4730dea995f64013a3936f9529b63580: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:09:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.540757 30168 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 1 FOLLOWER]: Refusing update from remote peer 4730dea995f64013a3936f9529b63580: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:09:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.542027 30534 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:09:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.542311 30529 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:09:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.559509 30539 mvcc.cc:204] Tried to move back new op lower bound from 7305267398810439680 to 7305267398127960064. Current Snapshot: MvccSnapshot[applied={T|T < 7305267398810439680}]
12:09:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.576611 30540 mvcc.cc:204] Tried to move back new op lower bound from 7305267398810439680 to 7305267398127960064. Current Snapshot: MvccSnapshot[applied={T|T < 7305267398810439680 or (T in {7305267398810439680})}]
12:09:08.666 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:08.668 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.195:41661
12:09:08.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:08.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.670527 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 30359
12:09:08.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.680652 30483 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:08.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:08.791220 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 30359
12:09:08.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:08.814 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] handling channelInactive
12:09:08.815 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state READY due to: connection closed
12:09:08.816 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: connection closed
12:09:08.829 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:08.830 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:08.830 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test)) after lookup
12:09:08.831 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:08.834 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.834 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.834 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.849 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:08.849 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:08.849 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [18ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [22ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [22ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [36ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663))) after lookup
12:09:08.850 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:08.852 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.852 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.852 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.889 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:08.889 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:08.890 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [18ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [22ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [22ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [36ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [38ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [40ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [40ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [76ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345))) after lookup
12:09:08.891 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:08.893 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.893 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.893 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.968 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:08.969 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:08.969 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=156), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [18ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [22ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [22ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [36ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [38ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [40ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [40ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [76ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [78ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [81ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [81ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [156ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218))) after lookup
12:09:08.970 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:08.972 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.972 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:08.972 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:09.149 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:09.150 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:09.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=337), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [18ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [22ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [22ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [36ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [38ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [40ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [40ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [76ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [78ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [81ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [81ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [156ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [158ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [160ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [160ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [333ms] refreshing cache from master, [334ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [337ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218) -> (continuation of Deferred@1967492134 after retry RPC@1299837030), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218) -> (continuation of Deferred@1967492134 after retry RPC@1299837030))) after lookup
12:09:09.151 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:09.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:09.153 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:09.153 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:09.510 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:09.510 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:09.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=698), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [18ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [22ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [22ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [36ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [38ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [40ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [40ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [76ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [78ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [81ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [81ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [156ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [158ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [160ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [160ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [333ms] refreshing cache from master, [334ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [337ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [339ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [341ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [341ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [693ms] refreshing cache from master, [693ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [697ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218) -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@1923987506 after retry RPC@1848848146), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218) -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@1923987506 after retry RPC@1848848146))) after lookup
12:09:09.512 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:09.513 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:09.514 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:09.514 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.110538 30555 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.110837 30555 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.110900 30556 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.111100 30556 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.112727 30556 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:10.114588 30240 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.115013 30555 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:10.116528 30102 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:10.118752 30240 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:10.119848 30102 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.124796 30168 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12" is_pre_election: true
12:09:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.125064 30168 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 1.
12:09:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.125605 30242 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.126044 30556 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Leader pre-election won for term 2
12:09:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.126202 30556 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Starting leader election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.126341 30556 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 1 FOLLOWER]: Advancing to term 2
12:09:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.126513 30306 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef" is_pre_election: true
12:09:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.128736 30556 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.129590 30306 raft_consensus.cc:2393] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5123b0bd0c44490288797ce5c01afd12 in current term 2: Already voted for candidate 5c2a68e64bbe44daa6a535cdad60daef in this term.
12:09:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.130630 30556 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 2 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.131314 30168 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.131561 30168 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 1 FOLLOWER]: Advancing to term 2
12:09:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.131336 30103 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12; no voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef
12:09:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:10.131817 30240 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.133756 30168 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 2.
12:09:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.133980 30555 raft_consensus.cc:2749] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
12:09:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.134320 30242 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.134749 30556 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Leader election won for term 2
12:09:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.135205 30556 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 LEADER]: Becoming Leader. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Running, Role: LEADER
12:09:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.135846 30556 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.140316 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef reported cstate change: term changed from 1 to 2, leader changed from 4730dea995f64013a3936f9529b63580 (127.29.27.195) to 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194). New cstate: current_term: 2 leader_uuid: "5c2a68e64bbe44daa6a535cdad60daef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: HEALTHY } } }
12:09:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.658183 30168 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Refusing update from remote peer 5c2a68e64bbe44daa6a535cdad60daef: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
12:09:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:10.659024 30577 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Connected to new peer: Peer: permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
12:09:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:10.674826 30240 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef -> Peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Couldn't send request to peer 4730dea995f64013a3936f9529b63580. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:12.209 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:12.209 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:12.210 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3397), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [3ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [18ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [22ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [22ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [36ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [38ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [40ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [40ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [76ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [78ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [81ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [81ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [156ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [158ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [160ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [160ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [333ms] refreshing cache from master, [334ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [337ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [339ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [341ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [341ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [693ms] refreshing cache from master, [693ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [697ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [700ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [702ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [702ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [3393ms] refreshing cache from master, [3393ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [3397ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@1628417268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@35d2b004 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218) -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@1923987506 after retry RPC@1848848146) -> (continuation of Deferred@847528775 after retry RPC@1169732026), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74942412 -> wakeup thread Time-limited test -> (continuation of Deferred@258703825 after retry RPC@144604663) -> (continuation of Deferred@369635754 after retry RPC@779114345) -> (continuation of Deferred@902531869 after retry RPC@595302218) -> (continuation of Deferred@1967492134 after retry RPC@1299837030) -> (continuation of Deferred@1923987506 after retry RPC@1848848146) -> (continuation of Deferred@847528775 after retry RPC@1169732026))) after lookup
12:09:12.211 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:12.213 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] Successfully connected to peer
12:09:12.239 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xb8633a5c, L:/127.0.0.1:54376 - R:/127.29.27.194:36885] using TOKEN/null
12:09:12.275 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.195:41661
12:09:12.276 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.278087 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.647476 30598 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.647889 30598 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.647989 30598 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.661396 30598 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.661646 30598 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.195
12:09:12.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.672513 30598 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:12.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.674863 30598 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.677400 30598 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.688424 30603 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.688611 30604 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:12.688812 30606 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.689437 30598 server_base.cc:1061] running on GCE node
12:09:12.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.690086 30598 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.691159 30598 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.692373 30598 hybrid_clock.cc:648] HybridClock initialized: now 1783512552692320 us; error 54 us; skew 500 ppm
12:09:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.692718 30598 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:12.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.695160 30598 webserver.cc:533] Webserver started at http://127.29.27.195:35881/ using document root <none> and password file <none>
12:09:12.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.696106 30598 fs_manager.cc:362] Metadata directory not provided
12:09:12.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.696215 30598 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.701932 30598 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
12:09:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.705065 30612 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.706596 30598 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
12:09:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.706841 30598 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.707801 30598 fs_report.cc:389] FS layout report
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/data
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.745173 30598 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.746184 30598 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.746672 30598 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.747794 30598 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:12.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.750051 30619 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.756806 30598 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.756980 30598 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.757131 30598 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.760900 30598 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.761062 30598 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.003s
12:09:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.761503 30619 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap starting.
12:09:12.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.785105 30619 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Log is configured to *not* fsync() on all Append() calls
12:09:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.804358 30619 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.805124 30619 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap complete.
12:09:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.806578 30619 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent bootstrapping tablet: real 0.045s	user 0.028s	sys 0.016s
12:09:12.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.816282 30619 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.817812 30619 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4730dea995f64013a3936f9529b63580, State: Initialized, Role: FOLLOWER
12:09:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.819245 30619 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.826033 30619 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent starting tablet: real 0.019s	user 0.014s	sys 0.003s
12:09:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.833359 30598 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.195:41661
12:09:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.833451 30729 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.195:41661 every 8 connection(s)
12:09:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.834957 30598 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.838781 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 30598
12:09:12.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:12.857 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:12.858 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.194:36885
12:09:12.859 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:09:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.860150 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 30223
12:09:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.905018 30731 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.905522 30731 heartbeater.cc:461] Registering TS with master...
12:09:12.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.906635 30731 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.909493 30346 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.910176 30732 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.910333 30732 heartbeater.cc:461] Registering TS with master...
12:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.910385 29848 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.910602 30730 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.910724 30730 heartbeater.cc:461] Registering TS with master...
12:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.910952 30732 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.911160 30730 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.912621 29920 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:12.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.916416 29991 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:12.917645 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.195:37955
12:09:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:13.016579 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 30223
12:09:13.036 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] handling channelInactive
12:09:13.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:13.036 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state READY due to: connection closed
12:09:13.037 [DEBUG - Time-limited test] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:13.038 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.038 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.038 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:13.048 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:13.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@339710759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@148ad57b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21891f58 -> wakeup thread Time-limited test)) after lookup
12:09:13.049 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:13.050 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.050 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.050 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:13.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:13.070 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [33ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@339710759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@148ad57b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21891f58 -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147))) after lookup
12:09:13.070 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:13.071 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.072 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.072 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:13.109 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:13.110 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [33ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [36ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@339710759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@148ad57b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21891f58 -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760))) after lookup
12:09:13.111 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:13.112 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.112 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.112 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:13.150 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:13.151 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=114), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [33ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [36ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [76ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [113ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@339710759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@148ad57b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760) -> (continuation of Deferred@141878215 after retry RPC@1958571283), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21891f58 -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760) -> (continuation of Deferred@141878215 after retry RPC@1958571283))) after lookup
12:09:13.151 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:13.153 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.153 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.153 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.330 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:13.330 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:13.331 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=294), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [33ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [36ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [76ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [113ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [115ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [117ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [117ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [289ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [293ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@339710759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@148ad57b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760) -> (continuation of Deferred@141878215 after retry RPC@1958571283) -> (continuation of Deferred@1803075775 after retry RPC@30397114), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21891f58 -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760) -> (continuation of Deferred@141878215 after retry RPC@1958571283) -> (continuation of Deferred@1803075775 after retry RPC@30397114))) after lookup
12:09:13.332 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:13.333 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.333 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:13.920585 30730 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.211270 30746 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.211526 30746 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.212320 30746 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:14.216437 30103 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:14.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.221700 30682 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "4730dea995f64013a3936f9529b63580" is_pre_election: true
12:09:14.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.222146 30682 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5123b0bd0c44490288797ce5c01afd12 in term 1.
12:09:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.223075 30102 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 5c2a68e64bbe44daa6a535cdad60daef
12:09:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.223459 30746 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Leader pre-election won for term 3
12:09:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.223600 30746 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.223737 30746 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 2 FOLLOWER]: Advancing to term 3
12:09:14.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.225958 30746 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:14.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.226671 30746 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 3 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.227324 30682 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "4730dea995f64013a3936f9529b63580"
12:09:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.227558 30682 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 1 FOLLOWER]: Advancing to term 3
12:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:14.228499 30103 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.231622 30682 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5123b0bd0c44490288797ce5c01afd12 in term 3.
12:09:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.232167 30102 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 5c2a68e64bbe44daa6a535cdad60daef
12:09:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.232504 30746 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 FOLLOWER]: Leader election won for term 3
12:09:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.232960 30746 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 LEADER]: Becoming Leader. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Running, Role: LEADER
12:09:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.233512 30746 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.237624 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 reported cstate change: term changed from 2 to 3, leader changed from 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194) to 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193). New cstate: current_term: 3 leader_uuid: "5123b0bd0c44490288797ce5c01afd12" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:14.650 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:14.650 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:14.651 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1614), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [33ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [36ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [36ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [76ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [113ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [115ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [117ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [117ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [289ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [293ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [295ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [297ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [297ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1609ms] refreshing cache from master, [1610ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1613ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@339710759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@148ad57b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760) -> (continuation of Deferred@141878215 after retry RPC@1958571283) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@942739035 after retry RPC@503437557), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21891f58 -> wakeup thread Time-limited test -> (continuation of Deferred@78560265 after retry RPC@1840517147) -> (continuation of Deferred@69158448 after retry RPC@1239857760) -> (continuation of Deferred@141878215 after retry RPC@1958571283) -> (continuation of Deferred@1803075775 after retry RPC@30397114) -> (continuation of Deferred@942739035 after retry RPC@503437557))) after lookup
12:09:14.652 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:14.653 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] Successfully connected to peer
12:09:14.674 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc93112a3, L:/127.0.0.1:54576 - R:/127.29.27.193:44909] using TOKEN/null
12:09:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.680723 30682 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Refusing update from remote peer 5123b0bd0c44490288797ce5c01afd12: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
12:09:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:14.680868 30103 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 -> Peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Couldn't send request to peer 5c2a68e64bbe44daa6a535cdad60daef. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:14.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.681788 30751 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
12:09:14.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.697898 30765 mvcc.cc:204] Tried to move back new op lower bound from 7305267423961780224 to 7305267422144446464. Current Snapshot: MvccSnapshot[applied={T|T < 7305267423961780224}]
12:09:14.723 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.194:36885
12:09:14.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:14.725456 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:36885
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39915
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.097576 30773 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.097973 30773 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.098098 30773 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.107189 30773 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.107437 30773 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.194
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.118206 30773 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:36885
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39915
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.120447 30773 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.122659 30773 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.133906 30779 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.134195 30778 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.136121 30773 server_base.cc:1061] running on GCE node
12:09:15.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.137601 30781 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.138548 30773 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.140093 30773 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.141366 30773 hybrid_clock.cc:648] HybridClock initialized: now 1783512555141321 us; error 37 us; skew 500 ppm
12:09:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.141800 30773 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.144585 30773 webserver.cc:533] Webserver started at http://127.29.27.194:39915/ using document root <none> and password file <none>
12:09:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.145500 30773 fs_manager.cc:362] Metadata directory not provided
12:09:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.145669 30773 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.152038 30773 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
12:09:15.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.155452 30787 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.157315 30773 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.157560 30773 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.158365 30773 fs_report.cc:389] FS layout report
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/data
12:09:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:15.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.176735 30773 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:15.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.177644 30773 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.177911 30773 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.178884 30773 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.181499 30794 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.188023 30773 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.188133 30773 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.188300 30773 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.191468 30773 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.191599 30773 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
12:09:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.192152 30794 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap starting.
12:09:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:15.220013 30103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111) [suppressed 3 similar messages]
12:09:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.224856 30794 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Log is configured to *not* fsync() on all Append() calls
12:09:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.244717 30794 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.245465 30794 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap complete.
12:09:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.247042 30794 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent bootstrapping tablet: real 0.055s	user 0.038s	sys 0.011s
12:09:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.255558 30794 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.256886 30794 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Initialized, Role: FOLLOWER
12:09:15.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.257946 30794 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:15.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.260106 30794 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent starting tablet: real 0.013s	user 0.016s	sys 0.000s
12:09:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.264501 30773 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.194:36885
12:09:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.264588 30904 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.194:36885 every 8 connection(s)
12:09:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.266160 30773 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:15.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.267428 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 30773
12:09:15.268 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:15.309 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:15.310 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.193:44909
12:09:15.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:09:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.311822 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 30086
12:09:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.322901 30208 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.332379 30907 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.332857 30907 heartbeater.cc:461] Registering TS with master...
12:09:15.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.334419 30907 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.336980 30906 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.337157 30906 heartbeater.cc:461] Registering TS with master...
12:09:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.337808 30906 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.339147 29920 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.339259 29849 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.340440 30905 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.340579 30905 heartbeater.cc:461] Registering TS with master...
12:09:15.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.341012 30905 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:15.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.342135 29990 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:15.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.343228 29990 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.194:44387
12:09:15.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:15.435722 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 30086
12:09:15.455 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] handling channelInactive
12:09:15.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:15.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state READY due to: connection closed
12:09:15.456 [DEBUG - Time-limited test] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:15.458 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.458 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.458 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.468 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:15.468 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:15.469 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test)) after lookup
12:09:15.469 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:15.470 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.470 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.470 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.489 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:15.489 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:15.489 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [14ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404))) after lookup
12:09:15.490 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:15.491 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.491 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.491 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.529 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:15.529 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:15.530 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [14ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [35ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [35ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145))) after lookup
12:09:15.530 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:15.532 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.532 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.532 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.568 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:15.569 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:15.569 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=113), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [14ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [35ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [35ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [76ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [112ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983))) after lookup
12:09:15.570 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:15.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.571 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:15.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:15.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=534), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [14ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [35ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [35ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [76ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [112ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [113ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [115ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [115ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [530ms] refreshing cache from master, [530ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [533ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@1896463377 after retry RPC@622140374), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@1896463377 after retry RPC@622140374))) after lookup
12:09:15.991 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:15.992 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.992 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:15.992 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.345700 30905 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:16.348 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:16.349 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:16.349 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=893), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [14ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [35ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [35ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [76ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [112ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [113ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [115ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [115ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [530ms] refreshing cache from master, [530ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [533ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [534ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [536ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [536ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [889ms] refreshing cache from master, [889ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [892ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@1896463377 after retry RPC@622140374) -> (continuation of Deferred@1386012103 after retry RPC@1608169449), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@1896463377 after retry RPC@622140374) -> (continuation of Deferred@1386012103 after retry RPC@1608169449))) after lookup
12:09:16.350 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:16.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:16.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:16.351 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:16.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.623451 30919 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 5123b0bd0c44490288797ce5c01afd12)
12:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.623812 30919 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.625550 30919 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:16.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:16.626684 30616 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:16.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:16.628530 30616 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:16.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.634137 30859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef" is_pre_election: true
12:09:16.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.634649 30859 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 2.
12:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.635586 30615 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 5123b0bd0c44490288797ce5c01afd12
12:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.636052 30919 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Leader pre-election won for term 4
12:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.636188 30919 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 5123b0bd0c44490288797ce5c01afd12)
12:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.636340 30919 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 3 FOLLOWER]: Advancing to term 4
12:09:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.639590 30919 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.640268 30919 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 4 election: Requested vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.640949 30859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.641175 30859 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 2 FOLLOWER]: Advancing to term 4
12:09:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:16.641819 30616 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:16.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.645148 30859 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 4.
12:09:16.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.645680 30615 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 5123b0bd0c44490288797ce5c01afd12
12:09:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.645991 30919 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 FOLLOWER]: Leader election won for term 4
12:09:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.646463 30919 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 LEADER]: Becoming Leader. State: Replica: 4730dea995f64013a3936f9529b63580, State: Running, Role: LEADER
12:09:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.646996 30919 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:16.651298 29990 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 reported cstate change: term changed from 3 to 4, leader changed from 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193) to 4730dea995f64013a3936f9529b63580 (127.29.27.195). New cstate: current_term: 4 leader_uuid: "4730dea995f64013a3936f9529b63580" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:17.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:17.095762 30859 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Refusing update from remote peer 4730dea995f64013a3936f9529b63580: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
12:09:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:17.097069 30919 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
12:09:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:17.113453 30616 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 -> Peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Couldn't send request to peer 5123b0bd0c44490288797ce5c01afd12. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:19.268 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:19.268 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:19.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3813), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [13ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [14ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [34ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [35ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [35ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [74ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [76ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [76ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [112ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [113ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [115ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [115ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [530ms] refreshing cache from master, [530ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [533ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [534ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [536ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [536ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [889ms] refreshing cache from master, [889ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [892ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [893ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [895ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [895ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [3809ms] refreshing cache from master, [3809ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [3812ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2074438500(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@6bf3c295 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@1896463377 after retry RPC@622140374) -> (continuation of Deferred@1386012103 after retry RPC@1608169449) -> (continuation of Deferred@1035577628 after retry RPC@572062679), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@150f307d -> wakeup thread Time-limited test -> (continuation of Deferred@1285916133 after retry RPC@541448404) -> (continuation of Deferred@1709460985 after retry RPC@1802087145) -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@1896463377 after retry RPC@622140374) -> (continuation of Deferred@1386012103 after retry RPC@1608169449) -> (continuation of Deferred@1035577628 after retry RPC@572062679))) after lookup
12:09:19.269 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:19.270 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] Successfully connected to peer
12:09:19.287 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1a4ff2c8, L:/127.0.0.1:45996 - R:/127.29.27.195:41661] using TOKEN/null
12:09:19.316 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.193:44909
12:09:19.316 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:09:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.317616 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:44909
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:19.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.686160 30616 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 -> Peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Couldn't send request to peer 5123b0bd0c44490288797ce5c01afd12. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:09:19.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.698066 30960 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:19.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.698446 30960 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:19.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.698536 30960 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.707686 30960 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.707876 30960 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.193
12:09:19.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.720502 30960 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:44909
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.722671 30960 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:19.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.725404 30960 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.736893 30965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:19.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.739108 30966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:19.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:19.739130 30968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.740208 30960 server_base.cc:1061] running on GCE node
12:09:19.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.740974 30960 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.742161 30960 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.743412 30960 hybrid_clock.cc:648] HybridClock initialized: now 1783512559743352 us; error 51 us; skew 500 ppm
12:09:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.743813 30960 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.746481 30960 webserver.cc:533] Webserver started at http://127.29.27.193:44091/ using document root <none> and password file <none>
12:09:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.747285 30960 fs_manager.cc:362] Metadata directory not provided
12:09:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.747400 30960 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:19.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.753331 30960 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.000s	sys 0.004s
12:09:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.756482 30974 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:19.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.758370 30960 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.758535 30960 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:05 on dist-test-slave-nb2j"
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.759253 30960 fs_report.cc:389] FS layout report
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/data
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.780405 30960 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.781235 30960 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.781510 30960 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:19.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.782454 30960 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.784960 30981 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:19.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.791646 30960 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:19.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.791818 30960 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:19.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.791942 30960 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.795516 30960 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:19.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.795714 30960 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.000s
12:09:19.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.796193 30981 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap starting.
12:09:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.821257 30981 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Log is configured to *not* fsync() on all Append() calls
12:09:19.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.841001 30981 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.841804 30981 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap complete.
12:09:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.843572 30981 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent bootstrapping tablet: real 0.048s	user 0.035s	sys 0.013s
12:09:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.851011 30981 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.852130 30981 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Initialized, Role: FOLLOWER
12:09:19.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.853427 30981 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.856266 30981 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent starting tablet: real 0.013s	user 0.004s	sys 0.007s
12:09:19.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.860904 30960 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.193:44909
12:09:19.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.860950 31089 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.193:44909 every 8 connection(s)
12:09:19.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.862426 30960 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:19.870 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.870370 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 30960
12:09:19.883 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:19.890 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.195:41661
12:09:19.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:19.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.891225 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 30598
12:09:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.895267 31092 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.895721 31092 heartbeater.cc:461] Registering TS with master...
12:09:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.896873 31092 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.899904 31091 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.900081 31091 heartbeater.cc:461] Registering TS with master...
12:09:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.900661 29919 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.900743 31091 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.902539 29849 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.903821 31090 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.903962 31090 heartbeater.cc:461] Registering TS with master...
12:09:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.904474 31090 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:19.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.905864 29991 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.907451 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.193:42775
12:09:19.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:19.919297 30725 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:20.022249 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 30598
12:09:20.041 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] handling channelInactive
12:09:20.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:20.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state READY due to: connection closed
12:09:20.042 [DEBUG - Time-limited test] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:20.043 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.043 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.043 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:20.048 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:20.049 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [7ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@610894098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@414a100b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@27119bd4 -> wakeup thread Time-limited test)) after lookup
12:09:20.049 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:20.050 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.050 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.050 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:20.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:20.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [7ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [9ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [9ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@610894098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@414a100b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@27119bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317))) after lookup
12:09:20.070 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:20.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.072 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:20.109 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:20.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [7ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [9ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [9ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [30ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [31ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@610894098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@414a100b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@27119bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896))) after lookup
12:09:20.110 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:20.111 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.111 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.112 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.228 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:20.229 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:20.229 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=187), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [7ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [9ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [9ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [30ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [31ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [69ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [70ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [71ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [187ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@610894098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@414a100b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896) -> (continuation of Deferred@1407109406 after retry RPC@1550252296), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@27119bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896) -> (continuation of Deferred@1407109406 after retry RPC@1550252296))) after lookup
12:09:20.230 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:20.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.231 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.232 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.708 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:20.709 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:20.710 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=669), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [7ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [9ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [9ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [30ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [31ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [69ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [70ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [71ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [187ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [189ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [191ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [191ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [664ms] refreshing cache from master, [664ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [667ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@610894098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@414a100b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896) -> (continuation of Deferred@1407109406 after retry RPC@1550252296) -> (continuation of Deferred@1603672711 after retry RPC@886245060), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@27119bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896) -> (continuation of Deferred@1407109406 after retry RPC@1550252296) -> (continuation of Deferred@1603672711 after retry RPC@886245060))) after lookup
12:09:20.712 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:20.713 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.713 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.713 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:20.910704 31090 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.302778 31104 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.303061 31104 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.304965 31104 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:21.306316 30789 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:21.308029 30789 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:21.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.313647 31044 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12" is_pre_election: true
12:09:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.314147 31044 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 3.
12:09:21.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.315348 30791 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:21.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.315922 31104 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Leader pre-election won for term 5
12:09:21.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.316051 31104 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Starting leader election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:21.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.316130 31104 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 4 FOLLOWER]: Advancing to term 5
12:09:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.318290 31104 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.319083 31104 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 5 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.319891 31044 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.320156 31044 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 3 FOLLOWER]: Advancing to term 5
12:09:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:21.320768 30789 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.324318 31044 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 5.
12:09:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.324877 30791 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.325294 31104 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 FOLLOWER]: Leader election won for term 5
12:09:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.325788 31104 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 LEADER]: Becoming Leader. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Running, Role: LEADER
12:09:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.326378 31104 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.330353 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef reported cstate change: term changed from 4 to 5, leader changed from 4730dea995f64013a3936f9529b63580 (127.29.27.195) to 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194). New cstate: current_term: 5 leader_uuid: "5c2a68e64bbe44daa6a535cdad60daef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: HEALTHY } } }
12:09:21.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:21.668 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:21.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1627), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [7ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [9ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [9ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [30ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [31ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [69ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [70ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [71ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [187ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [189ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [191ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [191ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [664ms] refreshing cache from master, [664ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [667ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [670ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [672ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [672ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [1624ms] refreshing cache from master, [1624ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1626ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@610894098(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@414a100b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896) -> (continuation of Deferred@1407109406 after retry RPC@1550252296) -> (continuation of Deferred@1603672711 after retry RPC@886245060) -> (continuation of Deferred@702862734 after retry RPC@86840144), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@27119bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1268839886 after retry RPC@1615071317) -> (continuation of Deferred@253668070 after retry RPC@1331174896) -> (continuation of Deferred@1407109406 after retry RPC@1550252296) -> (continuation of Deferred@1603672711 after retry RPC@886245060) -> (continuation of Deferred@702862734 after retry RPC@86840144))) after lookup
12:09:21.669 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:21.670 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] Successfully connected to peer
12:09:21.689 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb8d22794, L:/127.0.0.1:48418 - R:/127.29.27.194:36885] using TOKEN/null
12:09:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.695933 31044 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Refusing update from remote peer 5c2a68e64bbe44daa6a535cdad60daef: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
12:09:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:21.696223 30789 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef -> Peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Couldn't send request to peer 4730dea995f64013a3936f9529b63580. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:21.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.697021 31109 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Connected to new peer: Peer: permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
12:09:21.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.706720 31123 mvcc.cc:204] Tried to move back new op lower bound from 7305267452696322048 to 7305267451196747776. Current Snapshot: MvccSnapshot[applied={T|T < 7305267452696322048}]
12:09:21.730 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.195:41661
12:09:21.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:21.730998 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.100452 31132 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.100834 31132 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.100960 31132 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.109736 31132 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.110000 31132 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.195
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.121097 31132 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.123247 31132 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:22.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.125306 31132 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.136637 31140 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.136584 31138 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:22.136637 31137 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.138029 31132 server_base.cc:1061] running on GCE node
12:09:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.138664 31132 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.139773 31132 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.141036 31132 hybrid_clock.cc:648] HybridClock initialized: now 1783512562140963 us; error 61 us; skew 500 ppm
12:09:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.141443 31132 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:22.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.144202 31132 webserver.cc:533] Webserver started at http://127.29.27.195:35881/ using document root <none> and password file <none>
12:09:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.145078 31132 fs_manager.cc:362] Metadata directory not provided
12:09:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.145221 31132 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:22.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.151067 31132 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
12:09:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.154309 31146 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.155835 31132 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
12:09:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.156045 31132 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.156842 31132 fs_report.cc:389] FS layout report
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/data
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:22.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.197057 31132 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:22.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.198132 31132 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:22.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.198513 31132 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:22.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.199638 31132 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.201985 31153 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.208694 31132 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.208868 31132 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
12:09:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.209021 31132 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.212455 31132 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.212610 31132 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.001s
12:09:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.213205 31153 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap starting.
12:09:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.244483 31153 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Log is configured to *not* fsync() on all Append() calls
12:09:22.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.274766 31132 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.195:41661
12:09:22.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.275163 31260 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.195:41661 every 8 connection(s)
12:09:22.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.276808 31132 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:22.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.280009 31153 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.280952 31153 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap complete.
12:09:22.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.282979 31153 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent bootstrapping tablet: real 0.070s	user 0.057s	sys 0.005s
12:09:22.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:22.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.285239 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 31132
12:09:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.293027 31153 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.294464 31153 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4730dea995f64013a3936f9529b63580, State: Initialized, Role: FOLLOWER
12:09:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.295822 31153 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:22.323 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:22.324 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.194:36885
12:09:22.325 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:09:22.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.325708 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 30773
12:09:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.337602 31153 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent starting tablet: real 0.054s	user 0.035s	sys 0.012s
12:09:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.345980 31261 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.346488 31261 heartbeater.cc:461] Registering TS with master...
12:09:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.347836 31261 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.350008 30900 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.351531 29991 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:22.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.353097 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.195:59951
12:09:22.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.354295 31262 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:22.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.354460 31262 heartbeater.cc:461] Registering TS with master...
12:09:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.355196 31262 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.356523 31261 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.357626 29849 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.360603 31263 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.360767 31263 heartbeater.cc:461] Registering TS with master...
12:09:22.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.361162 31263 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:22.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.362993 29919 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:22.458020 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 30773
12:09:22.477 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:22.477 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] handling channelInactive
12:09:22.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state READY due to: connection closed
12:09:22.478 [DEBUG - Time-limited test] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:22.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.480 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.489 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:22.489 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:22.490 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test)) after lookup
12:09:22.490 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:22.492 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.492 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.492 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.508 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:22.508 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:22.509 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [12ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941))) after lookup
12:09:22.509 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:22.510 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.510 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.511 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.528 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:22.528 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:22.529 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [12ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [31ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [33ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [50ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515))) after lookup
12:09:22.529 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:22.530 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.531 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.569 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:22.569 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:22.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=92), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [12ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [31ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [33ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [50ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [51ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [53ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [91ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901))) after lookup
12:09:22.570 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:22.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.572 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.728 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:22.729 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:22.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=251), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [12ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [31ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [33ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [50ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [51ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [53ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [91ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [92ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [94ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [250ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901) -> (continuation of Deferred@609834364 after retry RPC@1314192279), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901) -> (continuation of Deferred@609834364 after retry RPC@1314192279))) after lookup
12:09:22.730 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:22.731 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.731 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:22.731 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:23.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:23.729 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:23.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1252), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [12ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [31ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [33ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [50ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [51ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [53ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [91ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [92ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [94ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [250ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [251ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [253ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [254ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1248ms] refreshing cache from master, [1248ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1251ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901) -> (continuation of Deferred@609834364 after retry RPC@1314192279) -> (continuation of Deferred@929705404 after retry RPC@1577840773), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901) -> (continuation of Deferred@609834364 after retry RPC@1314192279) -> (continuation of Deferred@929705404 after retry RPC@1577840773))) after lookup
12:09:23.730 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:23.731 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:23.731 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:23.731 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.798043 31278 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.798378 31278 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:23.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.800238 31278 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:23.802465 30977 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:23.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:23.804647 30977 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:23.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.810002 31215 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "4730dea995f64013a3936f9529b63580" is_pre_election: true
12:09:23.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.810564 31215 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5123b0bd0c44490288797ce5c01afd12 in term 4.
12:09:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.811695 30976 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 5c2a68e64bbe44daa6a535cdad60daef
12:09:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.812223 31278 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Leader pre-election won for term 6
12:09:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.812400 31278 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.812482 31278 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 5 FOLLOWER]: Advancing to term 6
12:09:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.814514 31278 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.815177 31278 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 6 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.815997 31215 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "4730dea995f64013a3936f9529b63580"
12:09:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.816289 31215 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 4 FOLLOWER]: Advancing to term 6
12:09:23.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:23.817328 30977 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.821028 31215 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5123b0bd0c44490288797ce5c01afd12 in term 6.
12:09:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.821616 30976 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 5c2a68e64bbe44daa6a535cdad60daef
12:09:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.821969 31278 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 FOLLOWER]: Leader election won for term 6
12:09:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.822439 31278 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 LEADER]: Becoming Leader. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Running, Role: LEADER
12:09:23.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.822952 31278 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:23.830534 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 reported cstate change: term changed from 5 to 6, leader changed from 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194) to 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193). New cstate: current_term: 6 leader_uuid: "5123b0bd0c44490288797ce5c01afd12" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:24.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:24.331498 31215 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Refusing update from remote peer 5123b0bd0c44490288797ce5c01afd12: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
12:09:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:24.332633 31296 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
12:09:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:24.353283 30977 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 -> Peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Couldn't send request to peer 5c2a68e64bbe44daa6a535cdad60daef. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:25.268 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:25.269 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:25.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2791), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [12ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [14ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [14ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [31ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [33ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [33ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [50ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [51ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [53ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [53ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [91ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [92ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [94ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [94ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [250ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [251ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [253ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [254ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1248ms] refreshing cache from master, [1248ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1251ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [1252ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1253ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1253ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2787ms] refreshing cache from master, [2787ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [2790ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@2094642599(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@515b51c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901) -> (continuation of Deferred@609834364 after retry RPC@1314192279) -> (continuation of Deferred@929705404 after retry RPC@1577840773) -> (continuation of Deferred@1170081384 after retry RPC@1144015234), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@40ef62c3 -> wakeup thread Time-limited test -> (continuation of Deferred@2145784760 after retry RPC@1143250941) -> (continuation of Deferred@367286637 after retry RPC@1969223515) -> (continuation of Deferred@354516461 after retry RPC@525251901) -> (continuation of Deferred@609834364 after retry RPC@1314192279) -> (continuation of Deferred@929705404 after retry RPC@1577840773) -> (continuation of Deferred@1170081384 after retry RPC@1144015234))) after lookup
12:09:25.270 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:25.271 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] Successfully connected to peer
12:09:25.291 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfc040de4, L:/127.0.0.1:43298 - R:/127.29.27.193:44909] using TOKEN/null
12:09:25.316 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.194:36885
12:09:25.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.319260 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:36885
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39915
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.663125 31317 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.663440 31317 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.663530 31317 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.673547 31317 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.673723 31317 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.194
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.684095 31317 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:36885
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39915
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.685896 31317 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.687898 31317 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.698438 31325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.698475 31324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:25.700769 31327 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.700947 31317 server_base.cc:1061] running on GCE node
12:09:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.701612 31317 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.702651 31317 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.703852 31317 hybrid_clock.cc:648] HybridClock initialized: now 1783512565703795 us; error 56 us; skew 500 ppm
12:09:25.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.704178 31317 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:25.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.706523 31317 webserver.cc:533] Webserver started at http://127.29.27.194:39915/ using document root <none> and password file <none>
12:09:25.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.707307 31317 fs_manager.cc:362] Metadata directory not provided
12:09:25.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.707401 31317 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:25.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.713554 31317 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
12:09:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.718762 31333 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.720796 31317 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.721022 31317 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.722079 31317 fs_report.cc:389] FS layout report
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/data
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:25.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.767390 31317 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.768352 31317 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.768625 31317 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.769613 31317 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.771948 31340 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:25.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.778184 31317 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:25.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.778362 31317 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
12:09:25.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.778520 31317 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:25.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.781736 31317 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.781878 31317 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
12:09:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.782357 31340 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap starting.
12:09:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.808727 31340 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Log is configured to *not* fsync() on all Append() calls
12:09:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.829910 31317 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.194:36885
12:09:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.830024 31447 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.194:36885 every 8 connection(s)
12:09:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.831473 31317 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.839892 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 31317
12:09:25.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:25.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.847574 31340 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:25.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.848462 31340 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap complete.
12:09:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.850251 31340 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent bootstrapping tablet: real 0.068s	user 0.037s	sys 0.021s
12:09:25.865 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:25.865 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.193:44909
12:09:25.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:09:25.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.866255 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 30960
12:09:25.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.873355 31340 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:25.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.874449 31340 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Initialized, Role: FOLLOWER
12:09:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.886862 31340 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.893108 31340 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent starting tablet: real 0.043s	user 0.011s	sys 0.001s
12:09:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.900714 31450 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.901126 31450 heartbeater.cc:461] Registering TS with master...
12:09:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.901798 31449 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.901988 31449 heartbeater.cc:461] Registering TS with master...
12:09:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.902370 31450 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.902602 31449 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.906900 29849 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.906901 29919 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:25.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.908367 31448 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:25.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.908532 31448 heartbeater.cc:461] Registering TS with master...
12:09:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.909083 31448 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:25.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.910789 29991 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.912349 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.194:35759
12:09:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.913628 31085 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:25.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:25.915767 31448 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:26.020722 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 30960
12:09:26.040 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] handling channelInactive
12:09:26.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:26.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state READY due to: connection closed
12:09:26.041 [DEBUG - Time-limited test] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:26.042 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.042 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.042 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:26.043133 31463 negotiation.cc:336] Failed RPC negotiation. Trace:
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.022117 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.29.27.193:41531 (local address 127.29.27.194:36885)
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.022838 (+   721us) server_negotiation.cc:207] Beginning negotiation
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.022850 (+    12us) server_negotiation.cc:400] Waiting for connection header
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.022889 (+    39us) server_negotiation.cc:408] Connection header received
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.023012 (+   123us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.023024 (+    12us) server_negotiation.cc:462] Received NEGOTIATE request from client
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.023314 (+   290us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.023538 (+   224us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:26.039571 (+ 16033us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.29.27.193:41531: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
12:09:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":461,"thread_start_us":318,"threads_started":1}
12:09:26.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:26.048 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:26.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@598936783(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@11af4272 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@521115b6 -> wakeup thread Time-limited test)) after lookup
12:09:26.049 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:26.050 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.050 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.050 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.068 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:26.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:26.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [10ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@598936783(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@11af4272 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@521115b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797))) after lookup
12:09:26.070 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:26.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.071 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:26.089 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:26.090 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [10ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [31ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [31ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [48ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@598936783(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@11af4272 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@521115b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924))) after lookup
12:09:26.090 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:26.092 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.092 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.092 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.129 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:26.129 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:26.130 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=89), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [10ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [31ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [31ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [48ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [50ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [52ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [52ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [89ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@598936783(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@11af4272 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@521115b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818))) after lookup
12:09:26.131 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:26.132 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.132 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.132 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:26.449 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:26.449 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=408), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [10ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [31ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [31ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [48ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [50ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [52ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [52ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [89ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [90ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [92ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [92ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [405ms] refreshing cache from master, [405ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [408ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@598936783(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@11af4272 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818) -> (continuation of Deferred@818815713 after retry RPC@1168808318), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@521115b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818) -> (continuation of Deferred@818815713 after retry RPC@1168808318))) after lookup
12:09:26.450 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:26.451 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.451 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:26.451 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.198901 31464 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 5123b0bd0c44490288797ce5c01afd12)
12:09:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.199154 31464 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.200804 31464 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:27.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.203855 31150 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.205551 31150 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:27.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.211414 31402 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef" is_pre_election: true
12:09:27.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.212273 31402 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 5.
12:09:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.213577 31149 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 5123b0bd0c44490288797ce5c01afd12
12:09:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.214073 31464 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Leader pre-election won for term 7
12:09:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.214218 31464 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 5123b0bd0c44490288797ce5c01afd12)
12:09:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.214306 31464 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 6 FOLLOWER]: Advancing to term 7
12:09:27.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.217514 31464 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:27.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.218396 31464 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 7 election: Requested vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:27.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.219070 31402 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:27.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.219340 31402 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 5 FOLLOWER]: Advancing to term 7
12:09:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.220467 31150 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:27.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.223476 31402 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 7.
12:09:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.224104 31149 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 5123b0bd0c44490288797ce5c01afd12
12:09:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.224675 31464 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 7 FOLLOWER]: Leader election won for term 7
12:09:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.225286 31464 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 7 LEADER]: Becoming Leader. State: Replica: 4730dea995f64013a3936f9529b63580, State: Running, Role: LEADER
12:09:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.225924 31464 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.230346 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 reported cstate change: term changed from 6 to 7, leader changed from 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193) to 4730dea995f64013a3936f9529b63580 (127.29.27.195). New cstate: current_term: 7 leader_uuid: "4730dea995f64013a3936f9529b63580" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:27.389 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:27.389 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:27.390 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1349), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [2ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [10ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [10ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [28ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [29ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [31ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [31ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [48ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [50ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [52ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [52ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [89ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [90ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [92ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [92ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [405ms] refreshing cache from master, [405ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [408ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [410ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [411ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [411ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [1345ms] refreshing cache from master, [1345ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1349ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@598936783(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@11af4272 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818) -> (continuation of Deferred@818815713 after retry RPC@1168808318) -> (continuation of Deferred@2044158861 after retry RPC@1755804832), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@521115b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1549358165 after retry RPC@113198797) -> (continuation of Deferred@1593032393 after retry RPC@579207924) -> (continuation of Deferred@1618203506 after retry RPC@1928395818) -> (continuation of Deferred@818815713 after retry RPC@1168808318) -> (continuation of Deferred@2044158861 after retry RPC@1755804832))) after lookup
12:09:27.390 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:27.392 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] Successfully connected to peer
12:09:27.425 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x0b88399a, L:/127.0.0.1:49864 - R:/127.29.27.195:41661] using TOKEN/null
12:09:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.431866 31402 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Refusing update from remote peer 4730dea995f64013a3936f9529b63580: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
12:09:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.432493 31150 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 -> Peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Couldn't send request to peer 5123b0bd0c44490288797ce5c01afd12. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.432842 31464 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
12:09:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.451893 31483 mvcc.cc:204] Tried to move back new op lower bound from 7305267476189782016 to 7305267475361480704. Current Snapshot: MvccSnapshot[applied={T|T < 7305267476189782016}]
12:09:27.471 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.193:44909
12:09:27.472 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.473397 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:44909
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.843950 31492 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.844416 31492 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.844556 31492 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.858964 31492 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.859258 31492 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.193
12:09:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.870432 31492 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:44909
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.872220 31492 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.874158 31492 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.885742 31500 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.885710 31497 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:27.885872 31498 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.886937 31492 server_base.cc:1061] running on GCE node
12:09:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.887876 31492 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.889109 31492 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.890424 31492 hybrid_clock.cc:648] HybridClock initialized: now 1783512567890370 us; error 42 us; skew 500 ppm
12:09:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.890847 31492 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:27.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.893836 31492 webserver.cc:533] Webserver started at http://127.29.27.193:44091/ using document root <none> and password file <none>
12:09:27.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.894718 31492 fs_manager.cc:362] Metadata directory not provided
12:09:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.894881 31492 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:27.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.901672 31492 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.003s
12:09:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.905364 31506 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:27.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.906945 31492 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.907174 31492 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:05 on dist-test-slave-nb2j"
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.908142 31492 fs_report.cc:389] FS layout report
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/data
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.929872 31492 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.930899 31492 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.931262 31492 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.932591 31492 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:27.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.934856 31513 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.941552 31492 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.941704 31492 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.941839 31492 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:27.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.945111 31492 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:27.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.945258 31492 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.002s	sys 0.000s
12:09:27.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.945874 31513 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap starting.
12:09:27.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.974152 31513 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Log is configured to *not* fsync() on all Append() calls
12:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.995776 31492 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.193:44909
12:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.995844 31620 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.193:44909 every 8 connection(s)
12:09:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:27.997602 31492 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.004149 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 31492
12:09:28.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:28.059 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:28.060 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.195:41661
12:09:28.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:28.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.061239 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 31132
12:09:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.072574 31623 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.073014 31623 heartbeater.cc:461] Registering TS with master...
12:09:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.074244 31623 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.075037 31622 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.075219 31622 heartbeater.cc:461] Registering TS with master...
12:09:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.075904 31622 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.077595 31513 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.077777 29919 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.077970 29849 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.078585 31513 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap complete.
12:09:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.080155 31621 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.080281 31513 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent bootstrapping tablet: real 0.135s	user 0.052s	sys 0.020s
12:09:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.080303 31621 heartbeater.cc:461] Registering TS with master...
12:09:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.081003 31621 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.082610 29991 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.084157 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.193:56221
12:09:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.085139 31256 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.087000 31513 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.088198 31513 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Initialized, Role: FOLLOWER
12:09:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.089185 31513 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.090565 31621 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.091262 31513 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent starting tablet: real 0.011s	user 0.007s	sys 0.003s
12:09:28.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:28.186995 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 31132
12:09:28.206 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] handling channelInactive
12:09:28.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:28.206 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state READY due to: connection closed
12:09:28.207 [DEBUG - Time-limited test] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:28.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.208 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.228 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:28.228 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:28.228 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@2b9d8b27 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test)) after lookup
12:09:28.229 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:28.230 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.230 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.248 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:28.249 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:28.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [23ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [24ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [42ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874))) after lookup
12:09:28.250 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:28.251 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.251 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.251 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.268 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:28.268 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:28.268 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [23ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [24ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [42ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [43ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [45ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [45ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [62ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115))) after lookup
12:09:28.269 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:28.270 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.270 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.270 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.328 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:28.328 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:28.329 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [23ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [24ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [42ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [43ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [45ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [45ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [62ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [63ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [64ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [122ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021))) after lookup
12:09:28.329 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:28.330 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.330 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.331 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.688 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:28.688 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:28.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=482), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [23ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [24ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [42ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [43ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [45ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [45ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [62ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [63ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [64ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [122ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [123ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [124ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [125ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [479ms] refreshing cache from master, [479ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [482ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016))) after lookup
12:09:28.689 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:28.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:28.691 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:29.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.553954 31639 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:29.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.554152 31639 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.555902 31639 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:29.557400 31335 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:29.559211 31335 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:29.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.562961 31576 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12" is_pre_election: true
12:09:29.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.563470 31576 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 6.
12:09:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.564021 31337 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.564436 31639 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Leader pre-election won for term 8
12:09:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.564584 31639 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Starting leader election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.564662 31639 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 7 FOLLOWER]: Advancing to term 8
12:09:29.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.566635 31639 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:29.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.567163 31639 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 8 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:29.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.567802 31576 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:29.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.567998 31576 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 6 FOLLOWER]: Advancing to term 8
12:09:29.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:29.568689 31335 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.571791 31576 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 8.
12:09:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.572208 31337 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.572544 31639 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 FOLLOWER]: Leader election won for term 8
12:09:29.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.572973 31639 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 LEADER]: Becoming Leader. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Running, Role: LEADER
12:09:29.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.573545 31639 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.577319 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef reported cstate change: term changed from 7 to 8, leader changed from 4730dea995f64013a3936f9529b63580 (127.29.27.195) to 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194). New cstate: current_term: 8 leader_uuid: "5c2a68e64bbe44daa6a535cdad60daef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: HEALTHY } } }
12:09:29.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:29.668 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:29.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1462), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [23ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [24ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [24ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [42ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [43ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [45ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [45ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [62ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [63ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [64ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [64ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [122ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [123ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [124ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [125ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [479ms] refreshing cache from master, [479ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [482ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [483ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [485ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [485ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [1459ms] refreshing cache from master, [1459ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1462ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016) -> (continuation of Deferred@1840925447 after retry RPC@486299108), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016) -> (continuation of Deferred@1840925447 after retry RPC@486299108))) after lookup
12:09:29.669 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:29.670 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] Successfully connected to peer
12:09:29.701 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xac059dfb, L:/127.0.0.1:52878 - R:/127.29.27.194:36885] using TOKEN/null
12:09:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.708033 31576 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Refusing update from remote peer 5c2a68e64bbe44daa6a535cdad60daef: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
12:09:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:29.708639 31335 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef -> Peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Couldn't send request to peer 4730dea995f64013a3936f9529b63580. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.709106 31644 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Connected to new peer: Peer: permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
12:09:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.726684 31657 mvcc.cc:204] Tried to move back new op lower bound from 7305267485511254016 to 7305267484976664576. Current Snapshot: MvccSnapshot[applied={T|T < 7305267485511254016}]
12:09:29.752 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.195:41661
12:09:29.752 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:29.753247 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:30.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.103327 31665 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:30.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.103693 31665 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:30.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.103798 31665 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:30.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.112557 31665 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:30.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.112727 31665 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.195
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.123210 31665 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:30.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.124831 31665 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:30.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.126525 31665 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.136765 31670 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.136803 31671 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.138089 31673 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.138559 31665 server_base.cc:1061] running on GCE node
12:09:30.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.139194 31665 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:30.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.140293 31665 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.141563 31665 hybrid_clock.cc:648] HybridClock initialized: now 1783512570141481 us; error 68 us; skew 500 ppm
12:09:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.141999 31665 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.144578 31665 webserver.cc:533] Webserver started at http://127.29.27.195:35881/ using document root <none> and password file <none>
12:09:30.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.145452 31665 fs_manager.cc:362] Metadata directory not provided
12:09:30.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.145592 31665 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:30.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.151870 31665 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.007s	sys 0.000s
12:09:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.155115 31679 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.156745 31665 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
12:09:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.156939 31665 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.157768 31665 fs_report.cc:389] FS layout report
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/data
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:30.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.182322 31665 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.183301 31665 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.183717 31665 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:30.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.184808 31665 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.187115 31686 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.193540 31665 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:30.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.193745 31665 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:30.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.193910 31665 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.197340 31665 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.197489 31665 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
12:09:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.197979 31686 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap starting.
12:09:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.225379 31686 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Log is configured to *not* fsync() on all Append() calls
12:09:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.253298 31665 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.195:41661
12:09:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.253351 31793 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.195:41661 every 8 connection(s)
12:09:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.255360 31665 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:30.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.263305 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 31665
12:09:30.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.274984 31686 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.275871 31686 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap complete.
12:09:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.277617 31686 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent bootstrapping tablet: real 0.080s	user 0.067s	sys 0.008s
12:09:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.282701 31686 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.283350 31686 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4730dea995f64013a3936f9529b63580, State: Initialized, Role: FOLLOWER
12:09:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.284075 31686 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.288116 31686 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent starting tablet: real 0.010s	user 0.010s	sys 0.002s
12:09:30.297 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:30.298 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.194:36885
12:09:30.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:09:30.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.298851 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 31317
12:09:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.315150 31796 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.315688 31796 heartbeater.cc:461] Registering TS with master...
12:09:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.317009 31796 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.320773 29919 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.327148 31795 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.327311 31794 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.327380 31795 heartbeater.cc:461] Registering TS with master...
12:09:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.327457 31794 heartbeater.cc:461] Registering TS with master...
12:09:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.328130 31795 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.328172 31794 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.330367 29849 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.331171 29990 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.332755 29990 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.195:40091
12:09:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.334185 31443 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.335536 31794 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.339257 31748 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 7 FOLLOWER]: Advancing to term 8
12:09:30.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:30.442193 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 31317
12:09:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:30.460824 31508 connection.cc:573] server connection from 127.29.27.194:51665 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
12:09:30.461 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] handling channelInactive
12:09:30.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:30.462 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state READY due to: connection closed
12:09:30.462 [DEBUG - Time-limited test] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:30.463 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.464 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.464 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.488 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:30.489 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:30.489 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test)) after lookup
12:09:30.490 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:30.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.491 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.508 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:30.508 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:30.509 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [27ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [29ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975))) after lookup
12:09:30.509 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:30.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.511 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:30.548 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:30.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=86), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [27ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [29ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [47ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [49ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [49ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [85ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540))) after lookup
12:09:30.548 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:30.549 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.549 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.549 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:30.689 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:30.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=228), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [27ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [29ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [47ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [49ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [49ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [85ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [86ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [87ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [87ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [227ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519))) after lookup
12:09:30.691 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:30.693 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.693 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.693 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.948 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:30.949 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:30.949 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=487), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [27ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [29ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [47ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [49ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [49ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [85ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [86ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [87ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [87ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [227ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [228ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [231ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [231ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [483ms] refreshing cache from master, [484ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [486ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1606151007 after retry RPC@162940939), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1606151007 after retry RPC@162940939))) after lookup
12:09:30.950 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:30.951 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.951 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:30.951 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:31.528 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:31.528 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:31.529 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1066), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [27ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [29ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [47ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [49ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [49ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [85ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [86ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [87ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [87ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [227ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [228ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [231ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [231ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [483ms] refreshing cache from master, [484ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [486ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [487ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [489ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [489ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1063ms] refreshing cache from master, [1063ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1066ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1606151007 after retry RPC@162940939) -> (continuation of Deferred@794524054 after retry RPC@337021542), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1606151007 after retry RPC@162940939) -> (continuation of Deferred@794524054 after retry RPC@337021542))) after lookup
12:09:31.530 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:31.531 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:31.531 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:31.532 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885
12:09:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.846356 31818 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.846591 31818 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.848364 31818 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:31.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:31.850126 31682 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:31.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:31.851629 31682 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.854549 31576 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12" is_pre_election: true
12:09:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.855396 31683 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4730dea995f64013a3936f9529b63580; no voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef
12:09:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.855756 31818 raft_consensus.cc:2749] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
12:09:31.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.943257 31823 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:31.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.943524 31823 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.945333 31823 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:31.946543 31509 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:31.948350 31509 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.952728 31748 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "4730dea995f64013a3936f9529b63580" is_pre_election: true
12:09:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.953033 31748 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5123b0bd0c44490288797ce5c01afd12 in term 8.
12:09:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.953529 31508 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 5c2a68e64bbe44daa6a535cdad60daef
12:09:31.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.953912 31823 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Leader pre-election won for term 9
12:09:31.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.954047 31823 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 5c2a68e64bbe44daa6a535cdad60daef)
12:09:31.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.954155 31823 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 8 FOLLOWER]: Advancing to term 9
12:09:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.956286 31823 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.957034 31823 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 9 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.957604 31748 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "4730dea995f64013a3936f9529b63580"
12:09:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.957849 31748 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 8 FOLLOWER]: Advancing to term 9
12:09:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:31.958779 31509 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111)
12:09:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.960225 31748 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5123b0bd0c44490288797ce5c01afd12 in term 9.
12:09:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.960650 31508 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5123b0bd0c44490288797ce5c01afd12; no voters: 5c2a68e64bbe44daa6a535cdad60daef
12:09:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.960989 31823 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Leader election won for term 9
12:09:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.961478 31823 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 LEADER]: Becoming Leader. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Running, Role: LEADER
12:09:31.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.962317 31823 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:31.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:31.966531 29990 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 reported cstate change: term changed from 8 to 9, leader changed from 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194) to 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193). New cstate: current_term: 9 leader_uuid: "5123b0bd0c44490288797ce5c01afd12" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:32.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:32.374926 31509 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 -> Peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Couldn't send request to peer 5c2a68e64bbe44daa6a535cdad60daef. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:32.477847 31748 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Refusing update from remote peer 5123b0bd0c44490288797ce5c01afd12: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
12:09:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:32.479058 31823 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 32, Time since last communication: 0.000s
12:09:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:34.860143 31509 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 -> Peer 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885): Couldn't send request to peer 5c2a68e64bbe44daa6a535cdad60daef. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.194:36885: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:09:35.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:35.049 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:35.049 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=4587), Traces: [0ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [2ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [2ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [27ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [29ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [29ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [47ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [49ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [49ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [85ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [86ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [87ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [87ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [227ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [228ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [231ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [231ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [483ms] refreshing cache from master, [484ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [486ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [487ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [489ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [489ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1063ms] refreshing cache from master, [1063ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1066ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [1068ms] sending RPC to server 5c2a68e64bbe44daa6a535cdad60daef, [1070ms] received response from server 5c2a68e64bbe44daa6a535cdad60daef: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [1070ms] delaying RPC due to: Network error: Failed to connect to peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885): Connection refused: /127.29.27.194:36885, [4583ms] refreshing cache from master, [4583ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [4586ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@866092031(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@4db2a31d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1606151007 after retry RPC@162940939) -> (continuation of Deferred@794524054 after retry RPC@337021542) -> (continuation of Deferred@1464829839 after retry RPC@2007659194), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3085b14c -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1606151007 after retry RPC@162940939) -> (continuation of Deferred@794524054 after retry RPC@337021542) -> (continuation of Deferred@1464829839 after retry RPC@2007659194))) after lookup
12:09:35.050 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:35.051 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] Successfully connected to peer
12:09:35.066 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xa823d669, L:/127.0.0.1:50024 - R:/127.29.27.193:44909] using TOKEN/null
12:09:35.096 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.194:36885
12:09:35.096 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.096940 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:36885
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39915
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:35.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.446998 31863 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:35.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.447324 31863 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:35.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.447409 31863 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.456156 31863 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.456334 31863 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.194
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.467039 31863 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.194:36885
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.194
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39915
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.194
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/logs
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.468799 31863 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.470613 31863 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.484386 31870 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.484426 31871 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:35.489283 31873 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.491137 31863 server_base.cc:1061] running on GCE node
12:09:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.492165 31863 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:35.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.493615 31863 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.497053 31863 hybrid_clock.cc:648] HybridClock initialized: now 1783512575496903 us; error 130 us; skew 500 ppm
12:09:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.497655 31863 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.504997 31863 webserver.cc:533] Webserver started at http://127.29.27.194:39915/ using document root <none> and password file <none>
12:09:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.506320 31863 fs_manager.cc:362] Metadata directory not provided
12:09:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.506525 31863 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.514956 31863 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.004s
12:09:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.518450 31879 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.520385 31863 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.520560 31863 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.521328 31863 fs_report.cc:389] FS layout report
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/wal
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/data
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:35.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.559248 31863 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.560328 31863 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.560693 31863 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.561789 31863 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.564445 31886 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.571205 31863 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.571360 31863 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.571513 31863 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.574888 31863 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.575037 31863 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
12:09:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.575507 31886 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap starting.
12:09:35.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.603472 31886 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Log is configured to *not* fsync() on all Append() calls
12:09:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.621902 31863 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.194:36885
12:09:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.621959 31993 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.194:36885 every 8 connection(s)
12:09:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.623936 31863 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-1/data/info.pb
12:09:35.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.626899 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 31863
12:09:35.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:35.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.658401 31886 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.659691 31886 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Bootstrap complete.
12:09:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.661306 31886 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent bootstrapping tablet: real 0.086s	user 0.066s	sys 0.016s
12:09:35.665 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:35.666 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.193:44909
12:09:35.666 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:09:35.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.667043 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 31492
12:09:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.674540 31886 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.681887 31886 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Initialized, Role: FOLLOWER
12:09:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.686076 31995 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.686601 31995 heartbeater.cc:461] Registering TS with master...
12:09:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.689145 31886 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.689646 31995 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.693341 29849 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.694087 31996 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.694268 31996 heartbeater.cc:461] Registering TS with master...
12:09:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.694928 31996 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:35.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.696954 29920 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.705376 31886 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef: Time spent starting tablet: real 0.044s	user 0.008s	sys 0.015s
12:09:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.706097 31616 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.707836 31994 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.708041 31994 heartbeater.cc:461] Registering TS with master...
12:09:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.708686 31994 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.710091 29991 ts_manager.cc:194] Re-registered known tserver with Master: 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.711759 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.194:57375
12:09:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.715282 31994 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:35.816224 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 31492
12:09:35.836 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:35.836 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] handling channelInactive
12:09:35.837 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state READY due to: connection closed
12:09:35.846 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:35.848 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.848 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.848 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.869 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:35.869 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:35.869 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=3, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [0ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [11ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [11ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@646585393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@21195f8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bdb82af -> wakeup thread Time-limited test)) after lookup
12:09:35.870 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:35.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.871 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.888 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:35.889 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:35.889 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=5, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [0ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [11ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [11ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [32ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [34ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [34ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@646585393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@21195f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bdb82af -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778))) after lookup
12:09:35.889 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:35.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.891 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.948 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:35.948 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:35.948 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=7, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=112), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [0ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [11ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [11ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [32ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [34ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [34ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [52ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [54ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [54ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [111ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@646585393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@21195f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bdb82af -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895))) after lookup
12:09:35.949 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:35.950 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.950 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:35.951 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:36.188 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:36.188 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:36.189 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=9, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=352), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [0ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [11ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [11ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [32ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [34ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [34ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [52ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [54ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [54ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [111ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [112ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [113ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [114ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [351ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@646585393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@21195f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895) -> (continuation of Deferred@1031936578 after retry RPC@1519042504), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bdb82af -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895) -> (continuation of Deferred@1031936578 after retry RPC@1519042504))) after lookup
12:09:36.189 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:36.190 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:36.190 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:36.190 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:36.708 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:36.709 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)[L],5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:36.709 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=11, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=872), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [0ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [11ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [11ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [32ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [34ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [34ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [52ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [54ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [54ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [111ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [112ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [113ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [114ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [351ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [352ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [353ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [353ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [868ms] refreshing cache from master, [868ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [871ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@646585393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@21195f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895) -> (continuation of Deferred@1031936578 after retry RPC@1519042504) -> (continuation of Deferred@1190830360 after retry RPC@1224815344), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bdb82af -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895) -> (continuation of Deferred@1031936578 after retry RPC@1519042504) -> (continuation of Deferred@1190830360 after retry RPC@1224815344))) after lookup
12:09:36.709 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] connecting to peer
12:09:36.710 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:36.710 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] cleaning up while in state CONNECTING due to: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:36.710 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909
12:09:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.137159 32009 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 5123b0bd0c44490288797ce5c01afd12)
12:09:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.137362 32009 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.138182 32009 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:37.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:37.139148 31683 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:37.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:37.140590 31683 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.146459 31949 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef" is_pre_election: true
12:09:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.147095 31949 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 8.
12:09:37.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.148267 31682 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 5123b0bd0c44490288797ce5c01afd12
12:09:37.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.148623 32009 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Leader pre-election won for term 10
12:09:37.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.148779 32009 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 5123b0bd0c44490288797ce5c01afd12)
12:09:37.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.148906 32009 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 9 FOLLOWER]: Advancing to term 10
12:09:37.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.151373 32009 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:37.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.152235 32009 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 10 election: Requested vote from peers 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:37.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.153029 31949 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "4730dea995f64013a3936f9529b63580" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef"
12:09:37.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.153277 31949 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 8 FOLLOWER]: Advancing to term 10
12:09:37.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:37.153990 31683 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111)
12:09:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.157305 31949 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4730dea995f64013a3936f9529b63580 in term 10.
12:09:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.157871 31682 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 5123b0bd0c44490288797ce5c01afd12
12:09:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.158313 32009 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 10 FOLLOWER]: Leader election won for term 10
12:09:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.158802 32009 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 10 LEADER]: Becoming Leader. State: Replica: 4730dea995f64013a3936f9529b63580, State: Running, Role: LEADER
12:09:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.159368 32009 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:37.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.163467 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 reported cstate change: term changed from 9 to 10, leader changed from 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193) to 4730dea995f64013a3936f9529b63580 (127.29.27.195). New cstate: current_term: 10 leader_uuid: "4730dea995f64013a3936f9529b63580" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: UNKNOWN } } }
12:09:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.690239 31949 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Refusing update from remote peer 4730dea995f64013a3936f9529b63580: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
12:09:37.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:37.691308 32027 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
12:09:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:37.709236 31683 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 -> Peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Couldn't send request to peer 5123b0bd0c44490288797ce5c01afd12. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:40.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:40.328075 31683 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 -> Peer 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909): Couldn't send request to peer 5123b0bd0c44490288797ce5c01afd12. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.193:44909: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:09:40.688 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:40.688 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:40.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=13, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=4852), Traces: [0ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [0ms] delaying RPC due to: Illegal state: connection is terminated, [8ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [11ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [11ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [32ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [34ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [34ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [52ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [54ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [54ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [111ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [112ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [113ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [114ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [351ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [352ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [353ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [353ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [868ms] refreshing cache from master, [868ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [871ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [872ms] sending RPC to server 5123b0bd0c44490288797ce5c01afd12, [873ms] received response from server 5123b0bd0c44490288797ce5c01afd12: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [873ms] delaying RPC due to: Network error: Failed to connect to peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909): Connection refused: /127.29.27.193:44909, [4848ms] refreshing cache from master, [4848ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [4851ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@646585393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@21195f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895) -> (continuation of Deferred@1031936578 after retry RPC@1519042504) -> (continuation of Deferred@1190830360 after retry RPC@1224815344) -> (continuation of Deferred@1760994636 after retry RPC@673225744), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bdb82af -> wakeup thread Time-limited test -> (continuation of Deferred@655740634 after retry RPC@616621778) -> (continuation of Deferred@1125481761 after retry RPC@1614865895) -> (continuation of Deferred@1031936578 after retry RPC@1519042504) -> (continuation of Deferred@1190830360 after retry RPC@1224815344) -> (continuation of Deferred@1760994636 after retry RPC@673225744))) after lookup
12:09:40.689 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:40.690 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] Successfully connected to peer
12:09:40.729 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x54fcce44, L:/127.0.0.1:32786 - R:/127.29.27.195:41661] using TOKEN/null
12:09:40.770 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.193:44909
12:09:40.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:40.771931 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:44909
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.123142 32054 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.123538 32054 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.123689 32054 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:41.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.132920 32054 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:41.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.133102 32054 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.193
12:09:41.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.143426 32054 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.193:44909
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.193
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.193
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/logs
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:41.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.145294 32054 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:41.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.147202 32054 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.157775 32059 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.158229 32060 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.158591 32062 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:41.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.159446 32054 server_base.cc:1061] running on GCE node
12:09:41.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.160142 32054 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:41.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.161173 32054 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:41.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.162401 32054 hybrid_clock.cc:648] HybridClock initialized: now 1783512581162303 us; error 92 us; skew 500 ppm
12:09:41.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.162707 32054 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.165180 32054 webserver.cc:533] Webserver started at http://127.29.27.193:44091/ using document root <none> and password file <none>
12:09:41.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.165917 32054 fs_manager.cc:362] Metadata directory not provided
12:09:41.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.166020 32054 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:41.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.171638 32054 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.001s
12:09:41.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.174713 32068 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.176277 32054 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.176429 32054 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:05 on dist-test-slave-nb2j"
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.177160 32054 fs_report.cc:389] FS layout report
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/wal
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/data
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:41.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.194118 32054 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.195003 32054 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.195259 32054 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:41.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.196238 32054 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.198773 32075 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:41.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.205156 32054 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:41.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.205286 32054 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:41.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.205395 32054 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:41.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.208521 32054 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:41.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.208652 32054 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
12:09:41.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.209192 32075 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap starting.
12:09:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.238950 32075 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Log is configured to *not* fsync() on all Append() calls
12:09:41.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.270743 32054 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.193:44909
12:09:41.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.270800 32182 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.193:44909 every 8 connection(s)
12:09:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.272449 32054 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-0/data/info.pb
12:09:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.272840 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 32054
12:09:41.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.291296 32075 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.292022 32075 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Bootstrap complete.
12:09:41.295 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:41.296 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.27.195:41661
12:09:41.296 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:41.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.297165 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 31665
12:09:41.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.314472 32075 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent bootstrapping tablet: real 0.106s	user 0.058s	sys 0.015s
12:09:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.325992 32183 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.326395 32183 heartbeater.cc:461] Registering TS with master...
12:09:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.327668 32183 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.327174 32075 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.333349 32075 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5123b0bd0c44490288797ce5c01afd12, State: Initialized, Role: FOLLOWER
12:09:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.334329 32189 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.334508 32189 heartbeater.cc:461] Registering TS with master...
12:09:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.334530 32075 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.335099 32189 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:41.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.337450 32187 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:41.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.337592 32187 heartbeater.cc:461] Registering TS with master...
12:09:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.338090 29991 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.338114 32187 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.338406 29919 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.339570 29849 ts_manager.cc:194] Re-registered known tserver with Master: 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.339710 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.193:34229
12:09:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.344436 32183 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.344485 31789 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.346568 32075 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12: Time spent starting tablet: real 0.032s	user 0.007s	sys 0.010s
12:09:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:41.453843 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 31665
12:09:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:41.472410 31881 connection.cc:573] server connection from 127.29.27.195:50675 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
12:09:41.473 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] handling channelInactive
12:09:41.473 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state READY due to: connection closed
12:09:41.473 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:41.474 [DEBUG - Time-limited test] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:41.475 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.475 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.475 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.488 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:41.489 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:41.489 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@934068851(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@1bc0088a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2331256a -> wakeup thread Time-limited test)) after lookup
12:09:41.489 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:41.490 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.490 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.491 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.508 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:41.508 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:41.509 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [16ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [18ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@934068851(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@1bc0088a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2331256a -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991))) after lookup
12:09:41.509 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:41.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.511 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:41.548 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:41.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [16ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [18ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [36ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [38ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [38ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@934068851(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@1bc0088a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2331256a -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955))) after lookup
12:09:41.549 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:41.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.550 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.589 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:41.589 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:41.589 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [16ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [18ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [36ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [38ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [38ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [75ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [77ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [77ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [112ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [115ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@934068851(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@1bc0088a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955) -> (continuation of Deferred@344401887 after retry RPC@1183865992), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2331256a -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955) -> (continuation of Deferred@344401887 after retry RPC@1183865992))) after lookup
12:09:41.590 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:41.591 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.591 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.591 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:41.748 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661)[L],5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)]}]
12:09:41.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=275), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [16ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [18ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [36ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [38ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [38ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [75ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [77ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [77ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [112ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [115ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [117ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [118ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [118ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [272ms] refreshing cache from master, [272ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [275ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@934068851(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@1bc0088a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955) -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2331256a -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955) -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650))) after lookup
12:09:41.749 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] connecting to peer
12:09:41.750 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.750 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:41.750 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4730dea995f64013a3936f9529b63580(127.29.27.195:41661) for tablet a061d38df9604855aecaa770f9a58714: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661
12:09:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.806304 32197 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:09:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.806660 32197 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:42.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.808643 32197 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194:36885)
12:09:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:42.810029 32070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:42.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:42.811928 32070 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:42.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.816301 31949 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5123b0bd0c44490288797ce5c01afd12" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "5c2a68e64bbe44daa6a535cdad60daef" is_pre_election: true
12:09:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.817386 32071 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12; no voters: 4730dea995f64013a3936f9529b63580, 5c2a68e64bbe44daa6a535cdad60daef
12:09:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.817888 32197 raft_consensus.cc:2749] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
12:09:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.955091 32202 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.955328 32202 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:42.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.957120 32202 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:42.958436 31881 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:42.960713 31881 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.966512 32138 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12" is_pre_election: true
12:09:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.966918 32138 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 9.
12:09:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.968072 31883 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.968583 32202 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Leader pre-election won for term 11
12:09:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.968724 32202 raft_consensus.cc:493] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Starting leader election (detected failure of leader 4730dea995f64013a3936f9529b63580)
12:09:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.968801 32202 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 10 FOLLOWER]: Advancing to term 11
12:09:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.970842 32202 raft_consensus.cc:515] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.971484 32202 leader_election.cc:290] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 11 election: Requested vote from peers 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661), 5123b0bd0c44490288797ce5c01afd12 (127.29.27.193:44909)
12:09:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.972173 32138 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a061d38df9604855aecaa770f9a58714" candidate_uuid: "5c2a68e64bbe44daa6a535cdad60daef" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "5123b0bd0c44490288797ce5c01afd12"
12:09:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.972409 32138 raft_consensus.cc:3060] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 9 FOLLOWER]: Advancing to term 11
12:09:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:42.973044 31881 leader_election.cc:336] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111)
12:09:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.976337 32138 raft_consensus.cc:2468] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c2a68e64bbe44daa6a535cdad60daef in term 11.
12:09:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.976853 31883 leader_election.cc:304] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5123b0bd0c44490288797ce5c01afd12, 5c2a68e64bbe44daa6a535cdad60daef; no voters: 4730dea995f64013a3936f9529b63580
12:09:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.977270 32202 raft_consensus.cc:2804] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 11 FOLLOWER]: Leader election won for term 11
12:09:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.977810 32202 raft_consensus.cc:697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [term 11 LEADER]: Becoming Leader. State: Replica: 5c2a68e64bbe44daa6a535cdad60daef, State: Running, Role: LEADER
12:09:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.978353 32202 consensus_queue.cc:237] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:42.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:42.982597 29991 catalog_manager.cc:5697] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef reported cstate change: term changed from 10 to 11, leader changed from 4730dea995f64013a3936f9529b63580 (127.29.27.195) to 5c2a68e64bbe44daa6a535cdad60daef (127.29.27.194). New cstate: current_term: 11 leader_uuid: "5c2a68e64bbe44daa6a535cdad60daef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } health_report { overall_health: HEALTHY } } }
12:09:43.008 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a061d38df9604855aecaa770f9a58714 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348' with partition [<start>, <end>)
12:09:43.008 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a061d38df9604855aecaa770f9a58714@[4730dea995f64013a3936f9529b63580(127.29.27.195:41661),5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909),5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)[L]]}]
12:09:43.009 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a061d38df9604855aecaa770f9a58714, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1535), Traces: [0ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [2ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [16ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [18ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [36ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [38ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [38ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [75ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [77ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [77ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [112ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [115ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [117ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [118ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [118ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [272ms] refreshing cache from master, [272ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [275ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, [276ms] sending RPC to server 4730dea995f64013a3936f9529b63580, [277ms] received response from server 4730dea995f64013a3936f9529b63580: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [277ms] delaying RPC due to: Network error: Failed to connect to peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661): Connection refused: /127.29.27.195:41661, [1532ms] refreshing cache from master, [1532ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.27.252:40279, [1535ms] Sub RPC GetTableLocations: received response from server master-127.29.27.252:40279: OK, deferred=Deferred@934068851(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$228/0x00007ff34c36b7a8@1bc0088a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955) -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2331256a -> wakeup thread Time-limited test -> (continuation of Deferred@1834755163 after retry RPC@1206666991) -> (continuation of Deferred@1426998517 after retry RPC@1642488955) -> (continuation of Deferred@344401887 after retry RPC@1183865992) -> (continuation of Deferred@1549328549 after retry RPC@409502650) -> (continuation of Deferred@2081388544 after retry RPC@539378024))) after lookup
12:09:43.009 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] connecting to peer
12:09:43.010 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] Successfully connected to peer
12:09:43.041 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x89bf21e6, L:/127.0.0.1:56716 - R:/127.29.27.194:36885] using TOKEN/null
12:09:43.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.047461 32138 raft_consensus.cc:1275] T a061d38df9604855aecaa770f9a58714 P 5123b0bd0c44490288797ce5c01afd12 [term 11 FOLLOWER]: Refusing update from remote peer 5c2a68e64bbe44daa6a535cdad60daef: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
12:09:43.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.047842 31881 consensus_peers.cc:597] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef -> Peer 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661): Couldn't send request to peer 4730dea995f64013a3936f9529b63580. Status: Network error: Client connection negotiation failed: client connection to 127.29.27.195:41661: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:09:43.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.048323 32202 consensus_queue.cc:1048] T a061d38df9604855aecaa770f9a58714 P 5c2a68e64bbe44daa6a535cdad60daef [LEADER]: Connected to new peer: Peer: permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
12:09:43.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.059149 32219 mvcc.cc:204] Tried to move back new op lower bound from 7305267540152029184 to 7305267539883491328. Current Snapshot: MvccSnapshot[applied={T|T < 7305267540152029184}]
12:09:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.060946 32220 mvcc.cc:204] Tried to move back new op lower bound from 7305267540152029184 to 7305267539883491328. Current Snapshot: MvccSnapshot[applied={T|T < 7305267540152029184}]
12:09:43.079 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.27.195:41661
12:09:43.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.080842 29807 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:09:43.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.431303 32226 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:09:43.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.431710 32226 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:09:43.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.431833 32226 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:09:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.440475 32226 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:09:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.440711 32226 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.27.195
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.451220 32226 tablet_server_runner.cc:78] Tablet server non-default flags:
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.27.212:42237
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:09:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.27.195:41661
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.27.195
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35881
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.27.254:34347,127.29.27.253:43153,127.29.27.252:40279
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.27.195
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/logs
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on bdcb31816ec0
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16856
12:09:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
12:09:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.453269 32226 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:09:43.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.455220 32226 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:09:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.466005 32231 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.466063 32232 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.466131 32234 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:09:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.466881 32226 server_base.cc:1061] running on GCE node
12:09:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.467563 32226 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:09:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.468706 32226 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:09:43.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.469928 32226 hybrid_clock.cc:648] HybridClock initialized: now 1783512583469887 us; error 38 us; skew 500 ppm
12:09:43.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.470230 32226 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:09:43.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.472620 32226 webserver.cc:533] Webserver started at http://127.29.27.195:35881/ using document root <none> and password file <none>
12:09:43.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.473426 32226 fs_manager.cc:362] Metadata directory not provided
12:09:43.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.473522 32226 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:09:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.479318 32226 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
12:09:43.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.482455 32240 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:09:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.484166 32226 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.484315 32226 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data,/tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4730dea995f64013a3936f9529b63580"
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:09:06 on dist-test-slave-nb2j"
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.485091 32226 fs_report.cc:389] FS layout report
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/wal
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/data
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:09:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:09:43.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.505765 32226 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:09:43.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.506639 32226 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:09:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.506896 32226 kserver.cc:163] Server-wide thread pool size limit: 3276
12:09:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.507944 32226 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:09:43.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.510138 32247 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:09:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.516691 32226 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:09:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.516816 32226 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
12:09:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.516923 32226 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:09:43.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.520416 32226 ts_tablet_manager.cc:616] Registered 1 tablets
12:09:43.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.520537 32226 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
12:09:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.521222 32247 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap starting.
12:09:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.551378 32247 log.cc:826] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Log is configured to *not* fsync() on all Append() calls
12:09:43.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.573405 32226 rpc_server.cc:307] RPC server started. Bound to: 127.29.27.195:41661
12:09:43.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.573467 32354 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.27.195:41661 every 8 connection(s)
12:09:43.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.575332 32226 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskkxu2RH/test-tmp/mini-kudu-cluster2634476697158383436/ts-2/data/info.pb
12:09:43.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.580519 29807 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu as pid 32226
12:09:43.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:09:43.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.610543 32247 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:09:43.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.611375 32247 tablet_bootstrap.cc:492] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Bootstrap complete.
12:09:43.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.613441 32247 ts_tablet_manager.cc:1403] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent bootstrapping tablet: real 0.093s	user 0.080s	sys 0.009s
12:09:43.618 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:43.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.625262 32247 raft_consensus.cc:359] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:43.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.626427 32247 raft_consensus.cc:740] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4730dea995f64013a3936f9529b63580, State: Initialized, Role: FOLLOWER
12:09:43.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.627597 32247 consensus_queue.cc:260] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4730dea995f64013a3936f9529b63580" member_type: VOTER last_known_addr { host: "127.29.27.195" port: 41661 } } peers { permanent_uuid: "5123b0bd0c44490288797ce5c01afd12" member_type: VOTER last_known_addr { host: "127.29.27.193" port: 44909 } } peers { permanent_uuid: "5c2a68e64bbe44daa6a535cdad60daef" member_type: VOTER last_known_addr { host: "127.29.27.194" port: 36885 } }
12:09:43.628 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512547348, tablet=a061d38df9604855aecaa770f9a58714, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:09:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.633754 32355 heartbeater.cc:344] Connected to a master server at 127.29.27.252:40279
12:09:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.634091 32355 heartbeater.cc:461] Registering TS with master...
12:09:43.634 [DEBUG - main] (Connection.java:575) [peer master-127.29.27.253:43153(127.29.27.253:43153)] disconnecting while in state READY
12:09:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.635114 32355 heartbeater.cc:507] Master 127.29.27.252:40279 requested a full tablet report, sending...
12:09:43.635 [DEBUG - main] (Connection.java:575) [peer 5123b0bd0c44490288797ce5c01afd12(127.29.27.193:44909)] disconnecting while in state TERMINATED
12:09:43.635 [DEBUG - main] (Connection.java:575) [peer master-127.29.27.254:34347(127.29.27.254:34347)] disconnecting while in state READY
12:09:43.635 [DEBUG - main] (Connection.java:575) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] disconnecting while in state READY
12:09:43.635 [DEBUG - main] (Connection.java:575) [peer 4730dea995f64013a3936f9529b63580(127.29.27.195:41661)] disconnecting while in state TERMINATED
12:09:43.636 [DEBUG - main] (Connection.java:575) [peer master-127.29.27.252:40279(127.29.27.252:40279)] disconnecting while in state READY
12:09:43.638 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.27.252:40279(127.29.27.252:40279)] handling channelInactive
12:09:43.638 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.29.27.253:43153(127.29.27.253:43153)] handling channelInactive
12:09:43.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.27.252:40279(127.29.27.252:40279)] cleaning up while in state READY due to: connection closed
12:09:43.639 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] handling channelInactive
12:09:43.639 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 5c2a68e64bbe44daa6a535cdad60daef(127.29.27.194:36885)] cleaning up while in state READY due to: connection closed
12:09:43.640 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
12:09:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.639909 29991 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:43.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.641487 29991 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.27.195:46865
12:09:43.643 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.27.254:34347(127.29.27.254:34347)] handling channelInactive
12:09:43.643 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.27.254:34347(127.29.27.254:34347)] cleaning up while in state READY due to: connection closed
12:09:43.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.643860 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 32054
12:09:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.648144 32357 heartbeater.cc:344] Connected to a master server at 127.29.27.253:43153
12:09:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.648319 32357 heartbeater.cc:461] Registering TS with master...
12:09:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.648713 32357 heartbeater.cc:507] Master 127.29.27.253:43153 requested a full tablet report, sending...
12:09:43.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.650508 29920 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:43.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.27.253:43153(127.29.27.253:43153)] cleaning up while in state READY due to: connection closed
12:09:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.668735 32356 heartbeater.cc:344] Connected to a master server at 127.29.27.254:34347
12:09:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.669006 32356 heartbeater.cc:461] Registering TS with master...
12:09:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.669390 32356 heartbeater.cc:507] Master 127.29.27.254:34347 requested a full tablet report, sending...
12:09:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.671046 29849 ts_manager.cc:194] Re-registered known tserver with Master: 4730dea995f64013a3936f9529b63580 (127.29.27.195:41661)
12:09:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.671283 32247 ts_tablet_manager.cc:1434] T a061d38df9604855aecaa770f9a58714 P 4730dea995f64013a3936f9529b63580: Time spent starting tablet: real 0.058s	user 0.015s	sys 0.017s
12:09:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.671844 32355 heartbeater.cc:499] Master 127.29.27.252:40279 was elected leader, sending a full tablet report...
12:09:43.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.679911 32178 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:43.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.787994 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 32054
12:09:43.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.806681 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 31863
12:09:43.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.814615 31989 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.933118 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 31863
12:09:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.953734 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 32226
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:09:43.955000 32373 negotiation.cc:336] Failed RPC negotiation. Trace:
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.933232 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.29.27.194:59707 (local address 127.29.27.195:41661)
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936145 (+  2913us) server_negotiation.cc:207] Beginning negotiation
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936152 (+     7us) server_negotiation.cc:400] Waiting for connection header
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936187 (+    35us) server_negotiation.cc:408] Connection header received
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936300 (+   113us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936307 (+     7us) server_negotiation.cc:462] Received NEGOTIATE request from client
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936550 (+   243us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.936782 (+   232us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 12:09:43.953457 (+ 16675us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.29.27.194:59707: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
12:09:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2768,"thread_start_us":2627,"threads_started":1}
12:09:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:43.963121 32350 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:44.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.058339 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 32226
12:09:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.076754 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 29818
12:09:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.078032 29879 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.196900 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 29818
12:09:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.219782 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 29886
12:09:44.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.221000 29950 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:44.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.337718 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 29886
12:09:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.361068 29807 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu pid 29957
12:09:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.362623 30021 generic_service.cc:196] Checking for leaks (request via RPC)
12:09:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:09:44.488384 29807 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskkxu2RH/build/asan/bin/kudu with pid 29957
12:09:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:09:44Z chronyd exiting

Time: 42.661

OK (1 test)

