JUnit version 4.13.2
.06:54:33.540 [INFO - main] (RandomUtils.java:49) Using random seed: 1786517673534
06:54:33.584 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
06:54:33.638 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:54:33.651 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoM2LY4/test-tmp
06:54:33.653 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
06:54:33.657 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoM2LY4/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
06:54:33.942 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

06:54:34.083 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:54:34.084 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

06:54:34.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.087648 21647 mini_oidc.cc:152] Starting JWKS server
06:54:34.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.090823 21647 webserver.cc:460] Webserver started at http://127.0.0.1:46445/ using document root <none> and password file <none>
06:54:34.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.091221 21647 mini_oidc.cc:189] Starting OIDC Discovery server
06:54:34.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.092079 21647 webserver.cc:460] Webserver started at http://127.0.0.1:33583/ using document root <none> and password file <none>
06:54:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T06:54:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:54:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T06:54:34Z Disabled control of system clock
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.126667 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/logs
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data/info.pb
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.254:44901
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.254
06:54:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:54:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46445/jwks.json with env {}
06:54:34.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.463475 21659 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:34.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.463888 21659 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:34.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.464035 21659 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.471762 21659 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:54:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.471984 21659 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.472066 21659 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:54:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.472134 21659 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:54:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.472186 21659 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:54:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.472259 21659 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46445/jwks.json
06:54:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.482375 21659 master_runner.cc:386] Master server non-default flags:
06:54:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data
06:54:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.254:44901
06:54:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46445/jwks.json
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data/info.pb
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.254
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/logs
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.484333 21659 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.486877 21659 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:34.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.500458 21667 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.500558 21665 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.500514 21664 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.501601 21659 server_base.cc:1034] running on GCE node
06:54:34.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.502485 21659 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:34.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.504268 21659 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:34.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.507635 21659 hybrid_clock.cc:648] HybridClock initialized: now 1786517674507071 us; error 555 us; skew 500 ppm
06:54:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.508134 21659 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.515372 21659 webserver.cc:460] Webserver started at http://127.21.35.254:39283/ using document root <none> and password file <none>
06:54:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.516299 21659 fs_manager.cc:359] Metadata directory not provided
06:54:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.516461 21659 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:34.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.516855 21659 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:54:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.519811 21659 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data/instance:
06:54:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "188c2a589a254c56b338b221f14cb0df"
06:54:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:34 on dist-test-slave-mvvj"
06:54:34.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.520853 21659 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal/instance:
06:54:34.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "188c2a589a254c56b338b221f14cb0df"
06:54:34.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:34 on dist-test-slave-mvvj"
06:54:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.526687 21659 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.004s	sys 0.001s
06:54:34.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.530768 21675 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.532317 21659 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
06:54:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.532539 21659 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal
06:54:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "188c2a589a254c56b338b221f14cb0df"
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:34 on dist-test-slave-mvvj"
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.532790 21659 fs_report.cc:383] FS layout report
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data/data
06:54:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.562086 21659 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:34.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.563202 21659 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:34.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.563616 21659 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:34.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.584946 21659 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.254:44901
06:54:34.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.584990 21732 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.254:44901 every 8 connection(s)
06:54:34.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.586597 21659 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/data/info.pb
06:54:34.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.590829 21733 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:54:34.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.597337 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 21659
06:54:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.597656 21647 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-0/wal/instance
06:54:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.597359 21733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } has no permanent_uuid. Determining permanent_uuid...
06:54:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.600147 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal
06:54:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data
06:54:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/logs
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data/info.pb
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:54:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:54:34.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.253:33133
06:54:34.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.253
06:54:34.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:34.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:34.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:54:34.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:34.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:54:34.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46445/jwks.json with env {}
06:54:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.619171 21733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } has no permanent_uuid. Determining permanent_uuid...
06:54:34.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.620684 21679 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.253:33133: connect: Connection refused (error 111)
06:54:34.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.622386 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.253:33133: Network error: Client connection negotiation failed: client connection to 127.21.35.253:33133: connect: Connection refused (error 111)
06:54:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.671931 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } attempt: 1
06:54:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.674358 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.253:33133: Network error: Client connection negotiation failed: client connection to 127.21.35.253:33133: connect: Connection refused (error 111)
06:54:34.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.742703 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } attempt: 2
06:54:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.745203 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.253:33133: Network error: Client connection negotiation failed: client connection to 127.21.35.253:33133: connect: Connection refused (error 111)
06:54:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.836499 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } attempt: 3
06:54:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.839243 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.253:33133: Network error: Client connection negotiation failed: client connection to 127.21.35.253:33133: connect: Connection refused (error 111)
06:54:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.957588 21738 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.957996 21738 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.958130 21738 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.966065 21738 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:54:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.966261 21738 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.966333 21738 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:54:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.966414 21738 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:54:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.966467 21738 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:54:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.966539 21738 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46445/jwks.json
06:54:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.977217 21738 master_runner.cc:386] Master server non-default flags:
06:54:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:54:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.253:33133
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46445/jwks.json
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data/info.pb
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.253
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/logs
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:54:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.979236 21738 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.982056 21738 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.982527 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } attempt: 4
06:54:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.984942 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.253:33133: Network error: Client connection negotiation failed: client connection to 127.21.35.253:33133: connect: Connection refused (error 111)
06:54:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.998567 21744 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.998698 21747 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:34.998735 21745 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:34.999934 21738 server_base.cc:1034] running on GCE node
06:54:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.001068 21738 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.002691 21738 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.003976 21738 hybrid_clock.cc:648] HybridClock initialized: now 1786517675003890 us; error 57 us; skew 500 ppm
06:54:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.004380 21738 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:35.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.007283 21738 webserver.cc:460] Webserver started at http://127.21.35.253:38543/ using document root <none> and password file <none>
06:54:35.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.008224 21738 fs_manager.cc:359] Metadata directory not provided
06:54:35.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.008368 21738 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:35.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.008797 21738 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:54:35.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.012080 21738 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data/instance:
06:54:35.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f610480c56174636ab7a3748a85950a1"
06:54:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:35 on dist-test-slave-mvvj"
06:54:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.013055 21738 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal/instance:
06:54:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f610480c56174636ab7a3748a85950a1"
06:54:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:35 on dist-test-slave-mvvj"
06:54:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.019133 21738 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.001s	sys 0.005s
06:54:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.023245 21753 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.024905 21738 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.003s
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.025127 21738 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f610480c56174636ab7a3748a85950a1"
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:35 on dist-test-slave-mvvj"
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.025398 21738 fs_report.cc:383] FS layout report
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal
06:54:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data/data
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:35.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:35.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.045917 21738 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.046985 21738 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.047392 21738 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.067667 21738 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.253:33133
06:54:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.067709 21807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.253:33133 every 8 connection(s)
06:54:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.069310 21738 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/data/info.pb
06:54:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.070977 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 21738
06:54:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.071172 21647 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-1/wal/instance
06:54:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.072294 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal
06:54:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data
06:54:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/logs
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data/info.pb
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.252:34003
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.252
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:54:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:54:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46445/jwks.json with env {}
06:54:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.073598 21808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:54:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.082696 21808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.100234 21808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.111315 21808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.112588 21755 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.114425 21808 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.163805 21808 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 1
06:54:35.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.166393 21808 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.234689 21808 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 2
06:54:35.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.237051 21808 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.285498 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } attempt: 5
06:54:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.292966 21733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.295369 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.328374 21808 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 3
06:54:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.330690 21808 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.346657 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 1
06:54:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.348770 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.417044 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 2
06:54:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.419231 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.473980 21808 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 4
06:54:35.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.476073 21808 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.525540 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 3
06:54:35.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.527626 21733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.35.252:34003: Network error: Client connection negotiation failed: client connection to 127.21.35.252:34003: connect: Connection refused (error 111)
06:54:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.536216 21810 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.536721 21810 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.536892 21810 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.554360 21810 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:54:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.554565 21810 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.554643 21810 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:54:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.554728 21810 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:54:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.554797 21810 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:54:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.554878 21810 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46445/jwks.json
06:54:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.576187 21810 master_runner.cc:386] Master server non-default flags:
06:54:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.252:34003
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46445/jwks.json
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data/info.pb
06:54:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.252
06:54:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/logs
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.578631 21810 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:35.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.581928 21810 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:35.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.597347 21820 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.597550 21821 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.598163 21823 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.599951 21810 server_base.cc:1034] running on GCE node
06:54:35.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.600997 21810 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:35.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.602640 21810 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.603984 21810 hybrid_clock.cc:648] HybridClock initialized: now 1786517675603902 us; error 53 us; skew 500 ppm
06:54:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.604465 21810 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:35.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.607187 21810 webserver.cc:460] Webserver started at http://127.21.35.252:42935/ using document root <none> and password file <none>
06:54:35.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.608065 21810 fs_manager.cc:359] Metadata directory not provided
06:54:35.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.608224 21810 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:35.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.608611 21810 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.611513 21810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data/instance:
06:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e40c0d0e66444eecb67ebe00932069dd"
06:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:35 on dist-test-slave-mvvj"
06:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.612491 21810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal/instance:
06:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e40c0d0e66444eecb67ebe00932069dd"
06:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:35 on dist-test-slave-mvvj"
06:54:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.618222 21810 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
06:54:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.622192 21830 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.623720 21810 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.623943 21810 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e40c0d0e66444eecb67ebe00932069dd"
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:35 on dist-test-slave-mvvj"
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.624133 21810 fs_report.cc:383] FS layout report
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data/data
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:35.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:35.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:35.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:35.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:35.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.658540 21810 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.659433 21810 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.659747 21810 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.680634 21810 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.252:34003
06:54:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.680672 21885 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.252:34003 every 8 connection(s)
06:54:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.682320 21810 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/data/info.pb
06:54:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.686331 21886 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:54:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.688647 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 21810
06:54:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.688807 21647 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/master-2/wal/instance
06:54:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.694756 21886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.704882 21733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 4
06:54:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.727172 21886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.733413 21733 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: Bootstrap starting.
06:54:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.736562 21886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } has no permanent_uuid. Determining permanent_uuid...
06:54:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.737941 21733 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: Neither blocks nor log segments found. Creating new log.
06:54:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.739076 21733 log.cc:824] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: Log is configured to *not* fsync() on all Append() calls
06:54:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.742647 21733 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: No bootstrap required, opened a new log
06:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.748324 21886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: Bootstrap starting.
06:54:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.752072 21886 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: Neither blocks nor log segments found. Creating new log.
06:54:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.752367 21733 raft_consensus.cc:348] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.752826 21733 raft_consensus.cc:374] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:54:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.752971 21733 raft_consensus.cc:729] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 188c2a589a254c56b338b221f14cb0df, State: Initialized, Role: FOLLOWER
06:54:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.753191 21886 log.cc:824] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: Log is configured to *not* fsync() on all Append() calls
06:54:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.753764 21733 consensus_queue.cc:260] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.756600 21886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: No bootstrap required, opened a new log
06:54:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.756646 21733 sys_catalog.cc:564] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: configured and running, proceeding with master startup.
06:54:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.757333 21896 sys_catalog.cc:455] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.767740 21896 sys_catalog.cc:458] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: This master's current role is: FOLLOWER
06:54:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.768379 21886 raft_consensus.cc:348] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.769233 21886 raft_consensus.cc:374] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:54:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.769568 21886 raft_consensus.cc:729] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e40c0d0e66444eecb67ebe00932069dd, State: Initialized, Role: FOLLOWER
06:54:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.770331 21886 consensus_queue.cc:260] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.771497 21904 sys_catalog.cc:455] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.772498 21904 sys_catalog.cc:458] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: This master's current role is: FOLLOWER
06:54:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.773139 21886 sys_catalog.cc:564] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: configured and running, proceeding with master startup.
06:54:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.775413 21808 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } attempt: 5
06:54:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.795790 21808 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: Bootstrap starting.
06:54:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.780689 21659 master_runner.cc:186] Error getting master registration for 127.21.35.253:33133: OK, instance_id { permanent_uuid: "f610480c56174636ab7a3748a85950a1" instance_seqno: 1786517675061873 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:54:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.804277 21808 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: Neither blocks nor log segments found. Creating new log.
06:54:35.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.805071 21659 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:54:35.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.806252 21808 log.cc:824] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: Log is configured to *not* fsync() on all Append() calls
06:54:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.808976 21911 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:54:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.809361 21911 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:54:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.814129 21808 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: No bootstrap required, opened a new log
06:54:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.817070 21681 tablet.cc:2369] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:54:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.829171 21925 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:54:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.829746 21925 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:54:35.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.830991 21808 raft_consensus.cc:348] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.832181 21808 raft_consensus.cc:374] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:54:35.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.820953 21810 master_runner.cc:186] Error getting master registration for 127.21.35.253:33133: OK, instance_id { permanent_uuid: "f610480c56174636ab7a3748a85950a1" instance_seqno: 1786517675061873 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:54:35.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.832453 21810 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:54:35.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.832788 21808 raft_consensus.cc:729] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f610480c56174636ab7a3748a85950a1, State: Initialized, Role: FOLLOWER
06:54:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.833709 21808 consensus_queue.cc:260] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.834836 21927 sys_catalog.cc:455] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.835469 21927 sys_catalog.cc:458] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: This master's current role is: FOLLOWER
06:54:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.836258 21808 sys_catalog.cc:564] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: configured and running, proceeding with master startup.
06:54:35.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.858001 21941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:54:35.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.858625 21941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:54:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.861277 21738 master_runner.cc:263] No leader master found from master f610480c56174636ab7a3748a85950a1
06:54:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.861444 21738 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:54:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.886408 21896 raft_consensus.cc:482] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:54:35.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.886847 21896 raft_consensus.cc:504] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.889137 21896 leader_election.cc:288] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f610480c56174636ab7a3748a85950a1 (127.21.35.253:33133), e40c0d0e66444eecb67ebe00932069dd (127.21.35.252:34003)
06:54:35.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.890691 21859 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "188c2a589a254c56b338b221f14cb0df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e40c0d0e66444eecb67ebe00932069dd" is_pre_election: true
06:54:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.891624 21859 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 188c2a589a254c56b338b221f14cb0df in term 0.
06:54:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.892393 21677 leader_election.cc:302] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 188c2a589a254c56b338b221f14cb0df, e40c0d0e66444eecb67ebe00932069dd; no voters: 
06:54:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.893555 21782 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "188c2a589a254c56b338b221f14cb0df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f610480c56174636ab7a3748a85950a1" is_pre_election: true
06:54:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.894271 21782 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 188c2a589a254c56b338b221f14cb0df in term 0.
06:54:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.898130 21896 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Leader pre-election won for term 1
06:54:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.898346 21896 raft_consensus.cc:482] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:54:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.898459 21896 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 0 FOLLOWER]: Advancing to term 1
06:54:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.901743 21896 raft_consensus.cc:504] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.903043 21782 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "188c2a589a254c56b338b221f14cb0df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f610480c56174636ab7a3748a85950a1"
06:54:35.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.903267 21782 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 0 FOLLOWER]: Advancing to term 1
06:54:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.907497 21782 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 188c2a589a254c56b338b221f14cb0df in term 1.
06:54:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.911692 21679 leader_election.cc:302] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 188c2a589a254c56b338b221f14cb0df, f610480c56174636ab7a3748a85950a1; no voters: 
06:54:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:35.912425 21836 tablet.cc:2369] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:54:35.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.921584 21859 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "188c2a589a254c56b338b221f14cb0df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e40c0d0e66444eecb67ebe00932069dd"
06:54:35.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.921840 21859 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 0 FOLLOWER]: Advancing to term 1
06:54:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.925920 21859 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 188c2a589a254c56b338b221f14cb0df in term 1.
06:54:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.929492 21896 leader_election.cc:288] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [CANDIDATE]: Term 1 election: Requested vote from peers f610480c56174636ab7a3748a85950a1 (127.21.35.253:33133), e40c0d0e66444eecb67ebe00932069dd (127.21.35.252:34003)
06:54:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.929903 21896 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 1 FOLLOWER]: Leader election won for term 1
06:54:35.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.931066 21896 raft_consensus.cc:686] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [term 1 LEADER]: Becoming Leader. State: Replica: 188c2a589a254c56b338b221f14cb0df, State: Running, Role: LEADER
06:54:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.932461 21896 consensus_queue.cc:237] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } }
06:54:35.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.946753 21943 sys_catalog.cc:455] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 188c2a589a254c56b338b221f14cb0df. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:35.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.947640 21943 sys_catalog.cc:458] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: This master's current role is: LEADER
06:54:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.956185 21946 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:54:35.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.959264 21946 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:54:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.981853 21859 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [term 1 FOLLOWER]: Refusing update from remote peer 188c2a589a254c56b338b221f14cb0df: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:54:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.983634 21943 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [LEADER]: Connected to new peer: Peer: permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:54:35.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.995793 21782 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [term 1 FOLLOWER]: Refusing update from remote peer 188c2a589a254c56b338b221f14cb0df: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:54:35.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:35.997748 21943 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [LEADER]: Connected to new peer: Peer: permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:54:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.032009 21927 sys_catalog.cc:455] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 188c2a589a254c56b338b221f14cb0df. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.032416 21927 sys_catalog.cc:458] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: This master's current role is: FOLLOWER
06:54:36.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.038595 21947 mvcc.cc:205] Tried to move back new op lower bound from 7317576400802881536 to 7317576400633802752. Current Snapshot: MvccSnapshot[applied={T|T < 7317576400802881536}]
06:54:36.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.042533 21949 mvcc.cc:205] Tried to move back new op lower bound from 7317576400802881536 to 7317576400633802752. Current Snapshot: MvccSnapshot[applied={T|T < 7317576400802881536}]
06:54:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.052207 21904 sys_catalog.cc:455] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 188c2a589a254c56b338b221f14cb0df. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.052676 21904 sys_catalog.cc:458] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: This master's current role is: FOLLOWER
06:54:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.065732 21948 mvcc.cc:205] Tried to move back new op lower bound from 7317576400802881536 to 7317576400633802752. Current Snapshot: MvccSnapshot[applied={T|T < 7317576400802881536}]
06:54:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.074739 21904 sys_catalog.cc:455] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.075018 21904 sys_catalog.cc:458] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd [sys.catalog]: This master's current role is: FOLLOWER
06:54:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.078097 21896 sys_catalog.cc:455] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.078456 21896 sys_catalog.cc:458] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: This master's current role is: LEADER
06:54:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.079131 21896 sys_catalog.cc:455] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.079376 21896 sys_catalog.cc:458] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df [sys.catalog]: This master's current role is: LEADER
06:54:36.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.086381 21946 catalog_manager.cc:1347] Generated new cluster ID: 702f9c27451c446993b4985c097a640c
06:54:36.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.086521 21946 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:54:36.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.088270 21927 sys_catalog.cc:455] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "188c2a589a254c56b338b221f14cb0df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "188c2a589a254c56b338b221f14cb0df" member_type: VOTER last_known_addr { host: "127.21.35.254" port: 44901 } } peers { permanent_uuid: "f610480c56174636ab7a3748a85950a1" member_type: VOTER last_known_addr { host: "127.21.35.253" port: 33133 } } peers { permanent_uuid: "e40c0d0e66444eecb67ebe00932069dd" member_type: VOTER last_known_addr { host: "127.21.35.252" port: 34003 } } }
06:54:36.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.088557 21927 sys_catalog.cc:458] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1 [sys.catalog]: This master's current role is: FOLLOWER
06:54:36.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.130105 21946 catalog_manager.cc:1370] Generated new certificate authority record
06:54:36.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.132745 21946 catalog_manager.cc:1504] Loading token signing keys...
06:54:36.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.160107 21946 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 188c2a589a254c56b338b221f14cb0df: Generated new TSK 0
06:54:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.161700 21946 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.191164 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:0
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.642055 21955 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.642442 21955 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.642582 21955 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.650621 21955 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:36.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.650904 21955 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.193
06:54:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.661322 21955 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:0
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:54:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:36.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.663395 21955 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.666031 21955 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.682101 21962 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.682020 21961 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:36.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:36.683283 21964 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:36.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.684031 21955 server_base.cc:1034] running on GCE node
06:54:36.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.684974 21955 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:36.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.686867 21955 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:36.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.688123 21955 hybrid_clock.cc:648] HybridClock initialized: now 1786517676688037 us; error 73 us; skew 500 ppm
06:54:36.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.688563 21955 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:36.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.691349 21955 webserver.cc:460] Webserver started at http://127.21.35.193:35499/ using document root <none> and password file <none>
06:54:36.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.692231 21955 fs_manager.cc:359] Metadata directory not provided
06:54:36.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.692399 21955 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:36.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.692787 21955 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:54:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.695801 21955 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/instance:
06:54:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:36 on dist-test-slave-mvvj"
06:54:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.697201 21955 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal/instance:
06:54:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:36 on dist-test-slave-mvvj"
06:54:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.703387 21955 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.001s
06:54:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.707790 21970 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:36.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.709702 21955 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.709933 21955 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:36 on dist-test-slave-mvvj"
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.710239 21955 fs_report.cc:383] FS layout report
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/data
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.733222 21955 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.734249 21955 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.734633 21955 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.735960 21955 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.738059 21955 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:54:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.738178 21955 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:54:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.738335 21955 ts_tablet_manager.cc:610] Registered 0 tablets
06:54:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.738400 21955 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:54:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.779776 21955 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.193:44873
06:54:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.779816 22095 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.193:44873 every 8 connection(s)
06:54:36.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.781292 21955 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:54:36.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.789232 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 21955
06:54:36.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.789497 21647 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal/instance
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.800905 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:0
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.814563 22098 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.814632 22097 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.814667 22096 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.815061 22097 heartbeater.cc:464] Registering TS with master...
06:54:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.816116 22097 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:36.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.815062 22098 heartbeater.cc:464] Registering TS with master...
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.815064 22096 heartbeater.cc:464] Registering TS with master...
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.817478 22096 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.817534 22098 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.818836 21694 ts_manager.cc:194] Registered new tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.819542 21846 ts_manager.cc:194] Registered new tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.819550 21770 ts_manager.cc:194] Registered new tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:54:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.820957 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.193:50475
06:54:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.832407 21925 catalog_manager.cc:1259] Loaded cluster ID: 702f9c27451c446993b4985c097a640c
06:54:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.832547 21925 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: loading cluster ID for follower catalog manager: success
06:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.835510 21925 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: acquiring CA information for follower catalog manager: success
06:54:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.837877 21925 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e40c0d0e66444eecb67ebe00932069dd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:54:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.861081 21941 catalog_manager.cc:1259] Loaded cluster ID: 702f9c27451c446993b4985c097a640c
06:54:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.861219 21941 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: loading cluster ID for follower catalog manager: success
06:54:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.863893 21941 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: acquiring CA information for follower catalog manager: success
06:54:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:36.865682 21941 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f610480c56174636ab7a3748a85950a1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:54:37.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.169615 22107 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:37.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.170029 22107 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:37.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.170137 22107 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:37.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.177789 22107 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:37.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.177976 22107 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.194
06:54:37.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.187544 22107 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:0
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.189137 22107 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:37.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.191342 22107 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.206480 22112 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.206470 22115 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.206507 22113 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.207768 22107 server_base.cc:1034] running on GCE node
06:54:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.208585 22107 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:37.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.209612 22107 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.210853 22107 hybrid_clock.cc:648] HybridClock initialized: now 1786517677210721 us; error 102 us; skew 500 ppm
06:54:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.211201 22107 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.217685 22107 webserver.cc:460] Webserver started at http://127.21.35.194:37059/ using document root <none> and password file <none>
06:54:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.218462 22107 fs_manager.cc:359] Metadata directory not provided
06:54:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.218582 22107 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:37.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.218935 22107 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:54:37.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.222078 22107 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/instance:
06:54:37.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:37.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.222884 22107 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal/instance:
06:54:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:37.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.228461 22107 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.007s	sys 0.000s
06:54:37.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.232633 22121 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.234226 22107 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
06:54:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.234496 22107 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.234754 22107 fs_report.cc:383] FS layout report
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/data
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.251664 22107 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.252743 22107 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.253134 22107 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.254308 22107 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.256120 22107 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:54:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.256245 22107 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:54:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.256449 22107 ts_tablet_manager.cc:610] Registered 0 tablets
06:54:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.256667 22107 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:54:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.298905 22107 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.194:44599
06:54:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.298954 22247 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.194:44599 every 8 connection(s)
06:54:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.300513 22107 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.302043 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 22107
06:54:37.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.302201 21647 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal/instance
06:54:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.308599 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:0
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.331039 22248 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.331542 22248 heartbeater.cc:464] Registering TS with master...
06:54:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.332238 22249 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:37.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.332402 22249 heartbeater.cc:464] Registering TS with master...
06:54:37.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.333244 22248 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:37.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.333269 22249 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.333977 22250 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.334127 22250 heartbeater.cc:464] Registering TS with master...
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.334605 22250 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.335181 21694 ts_manager.cc:194] Registered new tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.335567 21846 ts_manager.cc:194] Registered new tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.335652 21770 ts_manager.cc:194] Registered new tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.336426 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.194:40213
06:54:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.760242 22256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.760749 22256 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.760947 22256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.769526 22256 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.769804 22256 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.195
06:54:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.781419 22256 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:0
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:54:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:54:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.783277 22256 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.785650 22256 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.803220 22264 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:37.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.803377 22262 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:37.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:37.804030 22261 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.805006 22256 server_base.cc:1034] running on GCE node
06:54:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.805832 22256 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:37.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.806977 22256 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:37.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.808467 22256 hybrid_clock.cc:648] HybridClock initialized: now 1786517677808250 us; error 43 us; skew 500 ppm
06:54:37.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.808950 22256 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.812220 22256 webserver.cc:460] Webserver started at http://127.21.35.195:39365/ using document root <none> and password file <none>
06:54:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.813077 22256 fs_manager.cc:359] Metadata directory not provided
06:54:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.813350 22256 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.813795 22256 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:54:37.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.817149 22256 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/instance:
06:54:37.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:37.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.818126 22256 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal/instance:
06:54:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:37.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.824013 22097 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.825239 22256 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.005s
06:54:37.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.831527 22270 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:37.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.833446 22256 fs_manager.cc:725] Time spent opening block manager: real 0.006s	user 0.001s	sys 0.005s
06:54:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.833739 22256 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.834033 22256 fs_report.cc:383] FS layout report
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/data
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.869493 22256 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:37.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.871122 22256 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:37.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.871563 22256 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.872915 22256 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.875415 22256 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:54:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.875535 22256 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:54:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.875686 22256 ts_tablet_manager.cc:610] Registered 0 tablets
06:54:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.875753 22256 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:54:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.950600 22256 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.195:32955
06:54:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.950675 22396 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.195:32955 every 8 connection(s)
06:54:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.952174 22256 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:37.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.955963 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 22256
06:54:37.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.956167 21647 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal/instance
06:54:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.988876 22398 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.988955 22397 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.989239 22398 heartbeater.cc:464] Registering TS with master...
06:54:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.989246 22397 heartbeater.cc:464] Registering TS with master...
06:54:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.988925 22399 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.989768 22399 heartbeater.cc:464] Registering TS with master...
06:54:37.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.990808 22398 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:37.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.990881 22397 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:37.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.990917 22399 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:37.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.992620 21694 ts_manager.cc:194] Registered new tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.992918 21770 ts_manager.cc:194] Registered new tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.993726 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.195:56221
06:54:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:37.994692 21846 ts_manager.cc:194] Registered new tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:38.005800 21647 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:54:38.008 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:54:38.009 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

06:54:38.021 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.21.35.254"
      port: 44901
    }
    bound_http_address {
      host: "127.21.35.254"
      port: 39283
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.21.35.253"
      port: 33133
    }
    bound_http_address {
      host: "127.21.35.253"
      port: 38543
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.21.35.252"
      port: 34003
    }
    bound_http_address {
      host: "127.21.35.252"
      port: 42935
    }
  }
}

06:54:38.029 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

06:54:38.032 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.21.35.193"
      port: 44873
    }
    bound_http_address {
      host: "127.21.35.193"
      port: 35499
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.21.35.194"
      port: 44599
    }
    bound_http_address {
      host: "127.21.35.194"
      port: 37059
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.21.35.195"
      port: 32955
    }
    bound_http_address {
      host: "127.21.35.195"
      port: 39365
    }
  }
}

06:54:38.033 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:54:38.229 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
06:54:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:38.339624 22249 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:38.446 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.35.252:34003(127.21.35.252:34003)] connecting to peer
06:54:38.506 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.35.254:44901(127.21.35.254:44901)] connecting to peer
06:54:38.511 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.35.253:33133(127.21.35.253:33133)] connecting to peer
06:54:38.552 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.35.254:44901(127.21.35.254:44901)] Successfully connected to peer
06:54:38.561 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.35.252:34003(127.21.35.252:34003)] Successfully connected to peer
06:54:38.635 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.35.253:33133(127.21.35.253:33133)] Successfully connected to peer
06:54:38.662 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.35.252
06:54:38.662 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.35.254
06:54:38.771 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.35.253
06:54:38.908 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1ef3fefc, L:/127.0.0.1:36066 - R:/127.21.35.254:44901] using SASL/PLAIN
06:54:38.908 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x65f717f7, L:/127.0.0.1:59396 - R:/127.21.35.252:34003] using SASL/PLAIN
06:54:38.926 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa202be5d, L:/127.0.0.1:56948 - R:/127.21.35.253:33133] using SASL/PLAIN
06:54:38.954 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:54:38.956 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=556), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.35.252:34003, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.35.254:44901, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.35.253:33133, [518ms] Sub RPC ConnectToMaster: received response from server master-127.21.35.254:44901: OK, [518ms] Sub RPC ConnectToMaster: received response from server master-127.21.35.252:34003: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.21.35.253:33133: OK, deferred=null) after lookup
06:54:38.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:38.996942 22398 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.018172 21694 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36066:
06:54:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388"
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:54:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:54:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:39.020550 21694 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:54:39.073 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table cf8feae3900447e18593df554ebfba92
06:54:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.079254 22326 tablet_service.cc:1467] Processing CreateTablet for tablet ca7fe47bd75640edb87d395ae1fa4b98 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388 [id=cf8feae3900447e18593df554ebfba92]), partition=RANGE (key) PARTITION UNBOUNDED
06:54:39.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.081393 22326 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca7fe47bd75640edb87d395ae1fa4b98. 1 dirs total, 0 dirs full, 0 dirs failed
06:54:39.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.086338 22175 tablet_service.cc:1467] Processing CreateTablet for tablet ca7fe47bd75640edb87d395ae1fa4b98 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388 [id=cf8feae3900447e18593df554ebfba92]), partition=RANGE (key) PARTITION UNBOUNDED
06:54:39.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.088752 22175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca7fe47bd75640edb87d395ae1fa4b98. 1 dirs total, 0 dirs full, 0 dirs failed
06:54:39.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.095114 22024 tablet_service.cc:1467] Processing CreateTablet for tablet ca7fe47bd75640edb87d395ae1fa4b98 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388 [id=cf8feae3900447e18593df554ebfba92]), partition=RANGE (key) PARTITION UNBOUNDED
06:54:39.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.096758 22024 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca7fe47bd75640edb87d395ae1fa4b98. 1 dirs total, 0 dirs full, 0 dirs failed
06:54:39.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.106294 22433 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap starting.
06:54:39.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.108816 22434 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap starting.
06:54:39.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.111146 22433 tablet_bootstrap.cc:654] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Neither blocks nor log segments found. Creating new log.
06:54:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.112406 22434 tablet_bootstrap.cc:654] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Neither blocks nor log segments found. Creating new log.
06:54:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.112741 22433 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Log is configured to *not* fsync() on all Append() calls
06:54:39.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.113739 22434 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Log is configured to *not* fsync() on all Append() calls
06:54:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.114436 22436 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap starting.
06:54:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.117486 22434 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: No bootstrap required, opened a new log
06:54:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.117559 22433 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: No bootstrap required, opened a new log
06:54:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.117861 22433 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent bootstrapping tablet: real 0.012s	user 0.008s	sys 0.001s
06:54:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.117856 22434 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent bootstrapping tablet: real 0.009s	user 0.006s	sys 0.000s
06:54:39.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.119150 22436 tablet_bootstrap.cc:654] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Neither blocks nor log segments found. Creating new log.
06:54:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.120725 22436 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Log is configured to *not* fsync() on all Append() calls
06:54:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.129221 22436 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: No bootstrap required, opened a new log
06:54:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.129616 22436 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent bootstrapping tablet: real 0.015s	user 0.011s	sys 0.000s
06:54:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.130158 22433 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.130071 22434 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.130800 22433 raft_consensus.cc:374] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:54:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.130861 22434 raft_consensus.cc:374] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:54:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.130959 22433 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Initialized, Role: FOLLOWER
06:54:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.130987 22434 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Initialized, Role: FOLLOWER
06:54:39.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.131703 22434 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.131866 22433 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.134488 22433 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent starting tablet: real 0.017s	user 0.007s	sys 0.010s
06:54:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.142751 22434 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent starting tablet: real 0.025s	user 0.009s	sys 0.012s
06:54:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.146253 22436 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.146802 22436 raft_consensus.cc:374] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:54:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.146922 22436 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Initialized, Role: FOLLOWER
06:54:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.147646 22436 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.149902 22436 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent starting tablet: real 0.020s	user 0.014s	sys 0.003s
06:54:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.149897 22439 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:54:39.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.150774 22439 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.154090 22439 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.163307 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:54:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.163609 22346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3" is_pre_election: true
06:54:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.164068 22045 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 0.
06:54:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.164222 22346 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 0.
06:54:39.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.164983 22123 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 
06:54:39.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.165592 22439 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:54:39.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.165772 22439 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:54:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.165923 22439 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 0 FOLLOWER]: Advancing to term 1
06:54:39.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.168385 22439 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.169162 22439 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 1 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.169804 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.169839 22346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.170042 22346 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 0 FOLLOWER]: Advancing to term 1
06:54:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.170074 22045 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 0 FOLLOWER]: Advancing to term 1
06:54:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.172521 22346 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 1.
06:54:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.172637 22045 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 1.
06:54:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.173055 22125 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 
06:54:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.173388 22439 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 1 FOLLOWER]: Leader election won for term 1
06:54:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.174054 22439 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 1 LEADER]: Becoming Leader. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Running, Role: LEADER
06:54:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.174674 22439 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.180616 21692 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 reported cstate change: term changed from 0 to 1, leader changed from <none> to b006c851bd314c169fdce1bf12a65a51 (127.21.35.194). New cstate: current_term: 1 leader_uuid: "b006c851bd314c169fdce1bf12a65a51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:54:39.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table cf8feae3900447e18593df554ebfba92
06:54:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:39.213245 22400 tablet.cc:2369] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:54:39.230 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:39.238 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:39.240 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [10ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=null) after lookup
06:54:39.251 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:39.253 [DEBUG - kudu-nio-1] (Connection.java:757) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] Successfully connected to peer
06:54:39.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:39.287729 22099 tablet.cc:2369] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:54:39.310 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6877e202, L:/127.0.0.1:40928 - R:/127.21.35.194:44599] using TOKEN/null
06:54:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:39.321756 22255 tablet.cc:2369] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:54:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.322966 22045 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 1 FOLLOWER]: Refusing update from remote peer b006c851bd314c169fdce1bf12a65a51: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:54:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.322959 22346 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Refusing update from remote peer b006c851bd314c169fdce1bf12a65a51: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:54:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.324267 22439 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:54:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.324606 22444 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:54:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.344849 22446 mvcc.cc:205] Tried to move back new op lower bound from 7317576414494253056 to 7317576413903437824. Current Snapshot: MvccSnapshot[applied={T|T < 7317576414494253056}]
06:54:39.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.353063 22449 mvcc.cc:205] Tried to move back new op lower bound from 7317576414494253056 to 7317576413903437824. Current Snapshot: MvccSnapshot[applied={T|T < 7317576414494253056}]
06:54:39.456 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:39.461 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.194:44599
06:54:39.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:54:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.464166 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 22107
06:54:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.476089 22243 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:39.581346 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 22107
06:54:39.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:39.607 [DEBUG - kudu-nio-1] (Connection.java:239) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] handling channelInactive
06:54:39.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state READY due to: connection closed
06:54:39.609 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: connection closed
06:54:39.626 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:39.626 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:39.627 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test)) after lookup
06:54:39.628 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:39.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.631 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.646 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:39.646 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:39.647 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487))) after lookup
06:54:39.648 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:39.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.649 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.687 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:39.688 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:39.689 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [43ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [45ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [45ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [82ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648))) after lookup
06:54:39.690 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:39.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.693 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.725 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:39.726 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:39.727 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=121), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [43ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [45ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [45ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [82ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [86ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [89ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [89ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [120ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408))) after lookup
06:54:39.728 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:39.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.729 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.966 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:39.966 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:39.967 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=362), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [43ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [45ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [45ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [82ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [86ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [89ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [89ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [120ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [123ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [125ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [125ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [361ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820))) after lookup
06:54:39.968 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:39.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:39.970 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.286 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:40.286 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:40.287 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=681), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [43ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [45ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [45ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [82ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [86ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [89ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [89ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [120ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [123ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [125ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [125ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [361ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [363ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [366ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [366ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [677ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [681ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820) -> (continuation of Deferred@496814866 after retry RPC@1723531168), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820) -> (continuation of Deferred@496814866 after retry RPC@1723531168))) after lookup
06:54:40.288 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:40.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.290 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.706 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:40.707 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:40.708 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1102), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [43ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [45ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [45ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [82ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [86ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [89ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [89ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [120ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [123ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [125ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [125ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [361ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [363ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [366ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [366ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [677ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [681ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [684ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [686ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [686ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [1098ms] refreshing cache from master, [1098ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [1101ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820) -> (continuation of Deferred@496814866 after retry RPC@1723531168) -> (continuation of Deferred@132841502 after retry RPC@1343639858), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820) -> (continuation of Deferred@496814866 after retry RPC@1723531168) -> (continuation of Deferred@132841502 after retry RPC@1343639858))) after lookup
06:54:40.709 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:40.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.711 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.895196 22461 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.895485 22461 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:40.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.896085 22462 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:40.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.896495 22462 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:40.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.897537 22461 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:40.899994 22272 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.909066 22462 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:40.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.911001 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:54:40.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.911311 22045 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 1.
06:54:40.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:40.911643 22272 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:40.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:40.911957 21972 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:40.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.912178 22272 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:54:40.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.912674 22461 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Leader pre-election won for term 2
06:54:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.912817 22461 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Starting leader election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.912987 22461 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 1 FOLLOWER]: Advancing to term 2
06:54:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:40.914096 21972 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.915935 22461 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.916735 22461 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 2 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.917385 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.917646 22045 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 1 FOLLOWER]: Advancing to term 2
06:54:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.918206 22346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3" is_pre_election: true
06:54:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.918642 22346 raft_consensus.cc:2379] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 22bcd60c25a84b47b37128f3341b67b7 in current term 2: Already voted for candidate 57e078b7be9c46809c193a7d34c36ab3 in this term.
06:54:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:40.918962 22272 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.919526 21974 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7; no voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51
06:54:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.920637 22045 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 2.
06:54:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.920827 22462 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
06:54:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.921181 22272 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:54:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.921530 22461 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Leader election won for term 2
06:54:40.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.922187 22461 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 LEADER]: Becoming Leader. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Running, Role: LEADER
06:54:40.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.922765 22461 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:40.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:40.926527 21692 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 reported cstate change: term changed from 1 to 2, leader changed from b006c851bd314c169fdce1bf12a65a51 (127.21.35.194) to 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195). New cstate: current_term: 2 leader_uuid: "57e078b7be9c46809c193a7d34c36ab3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: HEALTHY } } }
06:54:41.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:41.388151 22272 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 -> Peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Couldn't send request to peer b006c851bd314c169fdce1bf12a65a51. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:54:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:41.464638 22045 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Refusing update from remote peer 57e078b7be9c46809c193a7d34c36ab3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
06:54:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:41.465819 22461 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
06:54:43.006 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:43.006 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:43.008 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3402), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [4ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [20ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [27ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [28ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [41ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [43ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [45ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [45ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [82ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [86ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [89ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [89ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [120ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [123ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [125ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [125ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [361ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [363ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [366ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [366ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [677ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [681ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [684ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [686ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [686ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [1098ms] refreshing cache from master, [1098ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [1101ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [1104ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [1107ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [1108ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3397ms] refreshing cache from master, [3397ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [3401ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1407818144(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@34d5fab8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820) -> (continuation of Deferred@496814866 after retry RPC@1723531168) -> (continuation of Deferred@132841502 after retry RPC@1343639858) -> (continuation of Deferred@193458492 after retry RPC@1769651903), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d1cb133 -> wakeup thread Time-limited test -> (continuation of Deferred@1550074345 after retry RPC@347727487) -> (continuation of Deferred@1787294039 after retry RPC@918796648) -> (continuation of Deferred@882267107 after retry RPC@523613408) -> (continuation of Deferred@1445739907 after retry RPC@754134820) -> (continuation of Deferred@496814866 after retry RPC@1723531168) -> (continuation of Deferred@132841502 after retry RPC@1343639858) -> (continuation of Deferred@193458492 after retry RPC@1769651903))) after lookup
06:54:43.009 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:43.010 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] Successfully connected to peer
06:54:43.032 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x155ea75f, L:/127.0.0.1:51034 - R:/127.21.35.195:32955] using TOKEN/null
06:54:43.061 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.194:44599
06:54:43.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.064513 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.433882 22500 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.434216 22500 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.434302 22500 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:43.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.442433 22500 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:43.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.442627 22500 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.194
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.452925 22500 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:43.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.454463 22500 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:43.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.457305 22500 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.477290 22505 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.477254 22506 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:43.477331 22508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.478304 22500 server_base.cc:1034] running on GCE node
06:54:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.479880 22500 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.484141 22500 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.485448 22500 hybrid_clock.cc:648] HybridClock initialized: now 1786517683485259 us; error 115 us; skew 500 ppm
06:54:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.485749 22500 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.493448 22500 webserver.cc:460] Webserver started at http://127.21.35.194:37059/ using document root <none> and password file <none>
06:54:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.494230 22500 fs_manager.cc:359] Metadata directory not provided
06:54:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.494380 22500 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:43.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.500635 22500 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.003s
06:54:43.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.504374 22516 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:43.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.505960 22500 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.002s
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.506127 22500 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.507228 22500 fs_report.cc:383] FS layout report
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/data
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:43.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.520076 22500 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.520951 22500 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.521225 22500 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.522349 22500 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:43.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.525146 22524 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:54:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.531601 22500 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:54:43.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.531773 22500 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:54:43.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.531924 22500 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:54:43.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.535221 22500 ts_tablet_manager.cc:610] Registered 1 tablets
06:54:43.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.535351 22500 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
06:54:43.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.535755 22524 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap starting.
06:54:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.564388 22524 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Log is configured to *not* fsync() on all Append() calls
06:54:43.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.586020 22524 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:54:43.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.587030 22524 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap complete.
06:54:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.589206 22524 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent bootstrapping tablet: real 0.054s	user 0.032s	sys 0.019s
06:54:43.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.592026 22500 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.194:44599
06:54:43.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.592120 22639 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.194:44599 every 8 connection(s)
06:54:43.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.593581 22500 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:43.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.602216 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 22500
06:54:43.603 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:43.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.607095 22524 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:43.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.614197 22524 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Initialized, Role: FOLLOWER
06:54:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.618278 22524 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.633827 22524 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent starting tablet: real 0.044s	user 0.017s	sys 0.008s
06:54:43.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.643143 22642 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:43.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.643589 22642 heartbeater.cc:464] Registering TS with master...
06:54:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.644974 22642 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:43.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.648133 21694 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.649638 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.194:34565
06:54:43.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.655287 22642 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:43.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.659680 22640 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:43.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.659907 22640 heartbeater.cc:464] Registering TS with master...
06:54:43.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.660305 22640 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.662290 21846 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:43.662 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:43.663 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.195:32955
06:54:43.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:54:43.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.664585 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 22256
06:54:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.666267 22643 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.666438 22643 heartbeater.cc:464] Registering TS with master...
06:54:43.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.666971 22643 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.669327 21771 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.687103 22391 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.688345 22591 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 1 FOLLOWER]: Advancing to term 2
06:54:43.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:43.806581 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 22256
06:54:43.827 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] handling channelInactive
06:54:43.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state READY due to: connection closed
06:54:43.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:43.828 [DEBUG - Time-limited test] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:43.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.830 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.845 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:43.846 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:43.847 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [16ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@177720072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@596cdc15 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@28639bc1 -> wakeup thread Time-limited test)) after lookup
06:54:43.848 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:43.850 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.850 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.866 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:43.866 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:43.867 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [16ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [20ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [22ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [22ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [37ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@177720072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@596cdc15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@28639bc1 -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121))) after lookup
06:54:43.868 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:43.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.870 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.885 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:43.885 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:43.886 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [16ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [20ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [22ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [22ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [37ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [42ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [56ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@177720072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@596cdc15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@28639bc1 -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962))) after lookup
06:54:43.888 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:43.890 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.890 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.890 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.966 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:43.966 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:43.967 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=138), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [16ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [20ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [22ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [22ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [37ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [42ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [56ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [60ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [62ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [62ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [137ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@177720072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@596cdc15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962) -> (continuation of Deferred@473953737 after retry RPC@1853158220), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@28639bc1 -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962) -> (continuation of Deferred@473953737 after retry RPC@1853158220))) after lookup
06:54:43.968 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:43.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:43.969 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:44.486 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:44.486 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:44.487 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=659), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [16ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [20ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [22ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [22ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [37ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [42ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [56ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [60ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [62ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [62ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [137ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [139ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [141ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [142ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [653ms] refreshing cache from master, [653ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [657ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@177720072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@596cdc15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962) -> (continuation of Deferred@473953737 after retry RPC@1853158220) -> (continuation of Deferred@453438093 after retry RPC@1377932338), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@28639bc1 -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962) -> (continuation of Deferred@473953737 after retry RPC@1853158220) -> (continuation of Deferred@453438093 after retry RPC@1377932338))) after lookup
06:54:44.488 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:44.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:44.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:44.490 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:45.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.161144 22664 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:54:45.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.161352 22664 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:45.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.162052 22664 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:45.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.165438 21974 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:45.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.169560 22591 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51" is_pre_election: true
06:54:45.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.170696 21972 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7; no voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51
06:54:45.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.171036 22664 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
06:54:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.199461 22670 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:54:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.199723 22670 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:45.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.201262 22670 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:45.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.202981 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.204705 22521 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:45.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.208604 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:54:45.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.208945 22045 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 2.
06:54:45.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.209455 22518 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51; no voters: 57e078b7be9c46809c193a7d34c36ab3
06:54:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.209852 22670 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 2 FOLLOWER]: Leader pre-election won for term 3
06:54:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.210021 22670 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:54:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.210172 22670 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 2 FOLLOWER]: Advancing to term 3
06:54:45.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.212522 22670 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:45.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.213212 22670 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 3 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.213791 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.214025 22045 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 2 FOLLOWER]: Advancing to term 3
06:54:45.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.215283 22521 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:45.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.216307 22045 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 3.
06:54:45.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.216750 22518 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51; no voters: 57e078b7be9c46809c193a7d34c36ab3
06:54:45.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.217074 22670 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 FOLLOWER]: Leader election won for term 3
06:54:45.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.217581 22670 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 LEADER]: Becoming Leader. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Running, Role: LEADER
06:54:45.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.218055 22670 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.222162 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 reported cstate change: term changed from 2 to 3, leader changed from 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195) to b006c851bd314c169fdce1bf12a65a51 (127.21.35.194). New cstate: current_term: 3 leader_uuid: "b006c851bd314c169fdce1bf12a65a51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:54:45.386 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:45.386 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:45.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1558), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [16ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [20ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [22ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [22ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [37ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [42ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [56ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [60ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [62ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [62ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [133ms] refreshing cache from master, [133ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [137ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [139ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [141ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [142ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [653ms] refreshing cache from master, [653ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [657ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [660ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [661ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [662ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [1553ms] refreshing cache from master, [1554ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [1557ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@177720072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@596cdc15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962) -> (continuation of Deferred@473953737 after retry RPC@1853158220) -> (continuation of Deferred@453438093 after retry RPC@1377932338) -> (continuation of Deferred@700980788 after retry RPC@816598444), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@28639bc1 -> wakeup thread Time-limited test -> (continuation of Deferred@1203040088 after retry RPC@1861263121) -> (continuation of Deferred@1936521897 after retry RPC@428800962) -> (continuation of Deferred@473953737 after retry RPC@1853158220) -> (continuation of Deferred@453438093 after retry RPC@1377932338) -> (continuation of Deferred@700980788 after retry RPC@816598444))) after lookup
06:54:45.388 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:45.390 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] Successfully connected to peer
06:54:45.409 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf1a79624, L:/127.0.0.1:41004 - R:/127.21.35.194:44599] using TOKEN/null
06:54:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.415609 22045 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Refusing update from remote peer b006c851bd314c169fdce1bf12a65a51: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
06:54:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.416245 22521 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 -> Peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Couldn't send request to peer 57e078b7be9c46809c193a7d34c36ab3. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:54:45.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.416818 22670 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
06:54:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.422130 22690 mvcc.cc:205] Tried to move back new op lower bound from 7317576439452635136 to 7317576438657257472. Current Snapshot: MvccSnapshot[applied={T|T < 7317576439452635136}]
06:54:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.425499 22691 mvcc.cc:205] Tried to move back new op lower bound from 7317576439452635136 to 7317576438657257472. Current Snapshot: MvccSnapshot[applied={T|T < 7317576439452635136}]
06:54:45.449 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.195:32955
06:54:45.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.451295 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:32955
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39365
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:45.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.821903 22697 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:45.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.822250 22697 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:45.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.822376 22697 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:45.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.831148 22697 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:45.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.831449 22697 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.195
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.841832 22697 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:32955
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39365
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.843700 22697 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.845943 22697 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.861474 22703 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.861503 22706 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:45.861398 22704 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.862625 22697 server_base.cc:1034] running on GCE node
06:54:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.863301 22697 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.864494 22697 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.865813 22697 hybrid_clock.cc:648] HybridClock initialized: now 1786517685865728 us; error 40 us; skew 500 ppm
06:54:45.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.866215 22697 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.869002 22697 webserver.cc:460] Webserver started at http://127.21.35.195:39365/ using document root <none> and password file <none>
06:54:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.869783 22697 fs_manager.cc:359] Metadata directory not provided
06:54:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.869982 22697 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.875932 22697 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
06:54:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.879122 22712 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.880692 22697 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.880924 22697 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.882085 22697 fs_report.cc:383] FS layout report
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/data
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:45.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.903120 22697 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.904286 22697 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.904668 22697 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.905930 22697 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:45.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.908581 22720 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:54:45.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.914603 22697 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:54:45.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.914765 22697 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:54:45.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.914930 22697 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:54:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.918681 22697 ts_tablet_manager.cc:610] Registered 1 tablets
06:54:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.918823 22697 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
06:54:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.919150 22720 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap starting.
06:54:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.945634 22720 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Log is configured to *not* fsync() on all Append() calls
06:54:45.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.968294 22720 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:54:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.969388 22720 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap complete.
06:54:45.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.972209 22720 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent bootstrapping tablet: real 0.053s	user 0.036s	sys 0.016s
06:54:45.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.980015 22697 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.195:32955
06:54:45.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.980137 22838 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.195:32955 every 8 connection(s)
06:54:45.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.981945 22697 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:45.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.987378 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 22697
06:54:45.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:45.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.988613 22720 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:45.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.990792 22720 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Initialized, Role: FOLLOWER
06:54:45.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:45.991905 22720 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:46.113 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:46.116 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.194:44599
06:54:46.117 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:54:46.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.117662 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 22500
06:54:46.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.129240 22720 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent starting tablet: real 0.157s	user 0.068s	sys 0.069s
06:54:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.141089 22841 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.141490 22841 heartbeater.cc:464] Registering TS with master...
06:54:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.142200 22842 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.142377 22842 heartbeater.cc:464] Registering TS with master...
06:54:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.142903 22841 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.143122 22842 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:46.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.145778 21694 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:46.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.146039 21771 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.147416 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.195:34629
06:54:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.149294 22839 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.149434 22839 heartbeater.cc:464] Registering TS with master...
06:54:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.149420 22635 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.149950 22839 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.151959 21847 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:46.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.153579 22841 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:46.257004 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 22500
06:54:46.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:46.278802 21973 connection.cc:535] server connection from 127.21.35.194:52469 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:54:46.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:46.279 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] handling channelInactive
06:54:46.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state READY due to: connection closed
06:54:46.280 [DEBUG - Time-limited test] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.282 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.282 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.282 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:46.306 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:46.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test)) after lookup
06:54:46.307 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.309 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:46.326 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:46.326 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [27ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [29ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [29ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [45ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251))) after lookup
06:54:46.327 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.328 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.345 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:46.346 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:46.347 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [27ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [29ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [29ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [45ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [46ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [48ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [48ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [65ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042))) after lookup
06:54:46.347 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.349 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.405 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:46.406 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:46.406 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=126), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [27ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [29ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [29ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [45ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [46ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [48ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [48ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [65ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [67ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [69ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [69ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [125ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075))) after lookup
06:54:46.407 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.408 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.426 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:46.426 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:46.427 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=147), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [27ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [29ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [29ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [45ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [46ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [48ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [48ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [65ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [67ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [69ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [69ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [125ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [127ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [128ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [128ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [145ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075) -> (continuation of Deferred@356252490 after retry RPC@538792055), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075) -> (continuation of Deferred@356252490 after retry RPC@538792055))) after lookup
06:54:46.428 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.431 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.584 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:46.585 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:46.585 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=305), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [27ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [29ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [29ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [45ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [46ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [48ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [48ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [65ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [67ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [69ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [69ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [125ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [127ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [128ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [128ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [145ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [147ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [151ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [151ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [301ms] refreshing cache from master, [301ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [304ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075) -> (continuation of Deferred@356252490 after retry RPC@538792055) -> (continuation of Deferred@1422317968 after retry RPC@504538519), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075) -> (continuation of Deferred@356252490 after retry RPC@538792055) -> (continuation of Deferred@1422317968 after retry RPC@504538519))) after lookup
06:54:46.586 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:46.587 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:46.588 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.690019 22857 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:54:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.690289 22857 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:47.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.692039 22857 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:47.694408 22714 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:47.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:47.696235 22714 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:47.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.700093 22045 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:54:47.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.701030 22714 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3; no voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51
06:54:47.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.701508 22857 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
06:54:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.758555 22862 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.758775 22862 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:47.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.759553 22862 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:47.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:47.760968 21972 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:54:47.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:47.762902 21972 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:47.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.769016 22791 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3" is_pre_election: true
06:54:47.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.769444 22791 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22bcd60c25a84b47b37128f3341b67b7 in term 2.
06:54:47.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.770537 21974 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:54:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.770875 22862 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Leader pre-election won for term 4
06:54:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.771050 22862 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Starting leader election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.771169 22862 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 3 FOLLOWER]: Advancing to term 4
06:54:47.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.773761 22862 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:47.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.774581 22862 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 4 election: Requested vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:47.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.775427 22791 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:47.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.775731 22791 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 2 FOLLOWER]: Advancing to term 4
06:54:47.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:47.776659 21972 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:47.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.780862 22791 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22bcd60c25a84b47b37128f3341b67b7 in term 4.
06:54:47.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.781409 21974 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:54:47.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.781910 22862 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Leader election won for term 4
06:54:47.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.782493 22862 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 LEADER]: Becoming Leader. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Running, Role: LEADER
06:54:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.783130 22862 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:47.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:47.789060 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 reported cstate change: term changed from 3 to 4, leader changed from b006c851bd314c169fdce1bf12a65a51 (127.21.35.194) to 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193). New cstate: current_term: 4 leader_uuid: "22bcd60c25a84b47b37128f3341b67b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:54:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:48.219137 21972 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 -> Peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Couldn't send request to peer b006c851bd314c169fdce1bf12a65a51. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:54:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:48.327677 22791 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Refusing update from remote peer 22bcd60c25a84b47b37128f3341b67b7: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
06:54:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:48.328574 22862 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
06:54:49.386 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:49.386 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:49.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3107), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [24ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [27ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [29ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [29ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [45ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [46ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [48ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [48ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [65ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [67ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [69ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [69ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [125ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [127ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [128ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [128ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [145ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [147ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [151ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [151ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [301ms] refreshing cache from master, [301ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [304ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [305ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [308ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [308ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3101ms] refreshing cache from master, [3102ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [3105ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@75861547(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@9c4c34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075) -> (continuation of Deferred@356252490 after retry RPC@538792055) -> (continuation of Deferred@1422317968 after retry RPC@504538519) -> (continuation of Deferred@1925791643 after retry RPC@395989802), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@699eccca -> wakeup thread Time-limited test -> (continuation of Deferred@118896903 after retry RPC@418467251) -> (continuation of Deferred@1299342758 after retry RPC@1791869042) -> (continuation of Deferred@1355043515 after retry RPC@1925768075) -> (continuation of Deferred@356252490 after retry RPC@538792055) -> (continuation of Deferred@1422317968 after retry RPC@504538519) -> (continuation of Deferred@1925791643 after retry RPC@395989802))) after lookup
06:54:49.388 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:49.390 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] Successfully connected to peer
06:54:49.412 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x37bbbae2, L:/127.0.0.1:34292 - R:/127.21.35.193:44873] using TOKEN/null
06:54:49.441 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.194:44599
06:54:49.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.443172 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.814579 22903 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.815034 22903 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.815135 22903 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.824843 22903 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.825057 22903 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.194
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.835268 22903 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:49.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.836823 22903 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:49.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.838989 22903 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.854061 22912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.854051 22910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:49.854475 22909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.855029 22903 server_base.cc:1034] running on GCE node
06:54:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.855822 22903 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:49.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.857164 22903 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.858443 22903 hybrid_clock.cc:648] HybridClock initialized: now 1786517689858345 us; error 62 us; skew 500 ppm
06:54:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.858773 22903 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.865880 22903 webserver.cc:460] Webserver started at http://127.21.35.194:37059/ using document root <none> and password file <none>
06:54:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.866655 22903 fs_manager.cc:359] Metadata directory not provided
06:54:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.866775 22903 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:49.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.873193 22903 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.001s
06:54:49.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.876710 22918 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:49.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.878218 22903 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
06:54:49.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.878624 22903 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:49.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:49.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.879736 22903 fs_report.cc:383] FS layout report
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/data
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:49.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.896430 22903 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.897357 22903 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.897648 22903 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.898773 22903 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.901171 22925 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:54:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.907217 22903 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:54:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.907393 22903 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:54:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.907516 22903 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:54:49.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.911252 22903 ts_tablet_manager.cc:610] Registered 1 tablets
06:54:49.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.911484 22903 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.000s	sys 0.001s
06:54:49.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.912250 22925 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap starting.
06:54:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.948832 22925 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Log is configured to *not* fsync() on all Append() calls
06:54:49.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.974758 22925 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:54:49.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.975513 22925 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap complete.
06:54:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.977341 22925 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent bootstrapping tablet: real 0.066s	user 0.045s	sys 0.018s
06:54:49.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.978505 22903 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.194:44599
06:54:49.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.978590 23040 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.194:44599 every 8 connection(s)
06:54:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.980172 22903 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.980700 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 22903
06:54:49.981 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.988940 22925 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.990448 22925 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Initialized, Role: FOLLOWER
06:54:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:49.991466 22925 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.000077 22925 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent starting tablet: real 0.023s	user 0.016s	sys 0.007s
06:54:50.021 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.043396 23044 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.043550 23047 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.043857 23047 heartbeater.cc:464] Registering TS with master...
06:54:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.043867 23044 heartbeater.cc:464] Registering TS with master...
06:54:50.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.045229 23047 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.047690 23044 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.048636 21771 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.049180 21694 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:50.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.050832 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.194:45235
06:54:50.054 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.193:44873
06:54:50.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:54:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.054965 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 21955
06:54:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.071158 23041 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.071352 23041 heartbeater.cc:464] Registering TS with master...
06:54:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.071938 23041 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.073709 21847 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.077618 22089 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:50.188714 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 21955
06:54:50.211 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] handling channelInactive
06:54:50.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state READY due to: connection closed
06:54:50.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:50.212 [DEBUG - Time-limited test] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:50.214 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.214 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.215 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.225 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:50.225 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:50.226 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [3ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [12ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@564521399(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@4e47645f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c40d08e -> wakeup thread Time-limited test)) after lookup
06:54:50.226 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:50.228 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.228 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.228 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.245 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:50.246 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:50.246 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [3ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [12ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [14ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [16ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [17ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [33ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@564521399(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@4e47645f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c40d08e -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429))) after lookup
06:54:50.247 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:50.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.248 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.286 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:50.286 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:50.287 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [3ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [12ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [14ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [16ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [17ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [33ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [36ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [36ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [73ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@564521399(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@4e47645f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c40d08e -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602))) after lookup
06:54:50.287 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:50.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.290 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:50.326 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:50.326 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=114), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [3ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [12ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [14ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [16ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [17ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [33ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [36ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [36ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [73ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [75ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [77ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [78ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [113ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@564521399(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@4e47645f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602) -> (continuation of Deferred@1642899430 after retry RPC@1642182751), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c40d08e -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602) -> (continuation of Deferred@1642899430 after retry RPC@1642182751))) after lookup
06:54:50.327 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:50.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.329 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.805 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:50.805 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:50.806 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=593), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [3ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [12ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [14ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [16ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [17ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [33ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [36ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [36ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [73ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [75ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [77ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [78ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [113ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [115ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [117ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [117ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [589ms] refreshing cache from master, [589ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [592ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@564521399(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@4e47645f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602) -> (continuation of Deferred@1642899430 after retry RPC@1642182751) -> (continuation of Deferred@1778889445 after retry RPC@1306493147), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c40d08e -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602) -> (continuation of Deferred@1642899430 after retry RPC@1642182751) -> (continuation of Deferred@1778889445 after retry RPC@1306493147))) after lookup
06:54:50.807 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:50.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:50.809 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.055001 23044 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.319808 23057 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 22bcd60c25a84b47b37128f3341b67b7)
06:54:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.320052 23057 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.320680 23057 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:51.323684 22714 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:54:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.330566 22992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51" is_pre_election: true
06:54:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.331072 22992 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 3.
06:54:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.332165 22714 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7
06:54:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.332504 23057 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Leader pre-election won for term 5
06:54:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.332657 23057 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 22bcd60c25a84b47b37128f3341b67b7)
06:54:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.332804 23057 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 4 FOLLOWER]: Advancing to term 5
06:54:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.335016 23057 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.335693 23057 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 5 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.336854 22992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.337143 22992 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 3 FOLLOWER]: Advancing to term 5
06:54:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:51.337796 22714 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:54:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.341862 22992 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 5.
06:54:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.342550 22714 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7
06:54:51.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.342983 23057 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Leader election won for term 5
06:54:51.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.343580 23057 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 LEADER]: Becoming Leader. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Running, Role: LEADER
06:54:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.344139 23057 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:51.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.348373 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 reported cstate change: term changed from 4 to 5, leader changed from 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193) to 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195). New cstate: current_term: 5 leader_uuid: "57e078b7be9c46809c193a7d34c36ab3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: HEALTHY } } }
06:54:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:51.869403 22714 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 -> Peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Couldn't send request to peer 22bcd60c25a84b47b37128f3341b67b7. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:54:51.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.928033 22992 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Refusing update from remote peer 57e078b7be9c46809c193a7d34c36ab3: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
06:54:51.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:51.929245 23077 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
06:54:52.344 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:52.345 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:52.345 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2133), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [3ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [12ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [14ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [16ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [17ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [33ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [36ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [36ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [73ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [75ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [77ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [78ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [113ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [115ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [117ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [117ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [589ms] refreshing cache from master, [589ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [592ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [594ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [597ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [598ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2129ms] refreshing cache from master, [2129ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [2132ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@564521399(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@4e47645f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602) -> (continuation of Deferred@1642899430 after retry RPC@1642182751) -> (continuation of Deferred@1778889445 after retry RPC@1306493147) -> (continuation of Deferred@227420726 after retry RPC@1013365252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c40d08e -> wakeup thread Time-limited test -> (continuation of Deferred@1074631154 after retry RPC@715390429) -> (continuation of Deferred@790290677 after retry RPC@1853742602) -> (continuation of Deferred@1642899430 after retry RPC@1642182751) -> (continuation of Deferred@1778889445 after retry RPC@1306493147) -> (continuation of Deferred@227420726 after retry RPC@1013365252))) after lookup
06:54:52.347 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:52.347 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] Successfully connected to peer
06:54:52.371 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x96ae4849, L:/127.0.0.1:43840 - R:/127.21.35.195:32955] using TOKEN/null
06:54:52.427 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.193:44873
06:54:52.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.428797 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:44873
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35499
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.802258 23095 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.802652 23095 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.802780 23095 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.811161 23095 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.811457 23095 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.193
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.822083 23095 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:44873
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:54:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35499
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:52.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:52.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:52.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.824123 23095 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.826133 23095 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.842800 23104 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.842895 23102 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.842828 23100 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:52.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.846016 23095 server_base.cc:1034] running on GCE node
06:54:52.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.846853 23095 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:52.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.847955 23095 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:52.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.849264 23095 hybrid_clock.cc:648] HybridClock initialized: now 1786517692849178 us; error 35 us; skew 500 ppm
06:54:52.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.849709 23095 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.852741 23095 webserver.cc:460] Webserver started at http://127.21.35.193:35499/ using document root <none> and password file <none>
06:54:52.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.853576 23095 fs_manager.cc:359] Metadata directory not provided
06:54:52.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.853768 23095 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.860123 23095 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
06:54:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.863687 23110 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.865434 23095 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:54:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.865676 23095 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:36 on dist-test-slave-mvvj"
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.866890 23095 fs_report.cc:383] FS layout report
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/data
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:52.873080 22714 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111) [suppressed 4 similar messages]
06:54:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.901304 23095 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.902302 23095 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.902683 23095 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:52.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.903995 23095 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:52.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.906749 23123 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:54:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.912938 23095 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:54:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.913130 23095 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:54:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.913271 23095 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:54:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.916635 23095 ts_tablet_manager.cc:610] Registered 1 tablets
06:54:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.916940 23095 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.000s	sys 0.003s
06:54:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.917311 23123 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap starting.
06:54:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.953218 23123 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Log is configured to *not* fsync() on all Append() calls
06:54:52.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.984835 23123 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:54:52.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.985754 23123 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap complete.
06:54:52.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:52.987761 23123 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent bootstrapping tablet: real 0.071s	user 0.050s	sys 0.016s
06:54:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.000036 23123 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.001641 23123 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Initialized, Role: FOLLOWER
06:54:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.002668 23123 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.010411 23123 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent starting tablet: real 0.022s	user 0.014s	sys 0.006s
06:54:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.013302 23095 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.193:44873
06:54:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.013321 23246 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.193:44873 every 8 connection(s)
06:54:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.015300 23095 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:54:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.025795 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 23095
06:54:53.026 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:53.032 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:97)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
06:54:53.033 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:79)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
06:54:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.042200 23248 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.042593 23248 heartbeater.cc:464] Registering TS with master...
06:54:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.043097 23249 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.043232 23249 heartbeater.cc:464] Registering TS with master...
06:54:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.043992 23248 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.044003 23249 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:53.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.047529 21771 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:54:53.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.047984 23247 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:53.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.048146 23247 heartbeater.cc:464] Registering TS with master...
06:54:53.048 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:53.049 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.195:32955
06:54:53.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:54:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.048728 23247 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.049578 21694 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:54:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.050513 21847 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:54:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.051285 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.193:56715
06:54:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.051945 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 22697
06:54:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.063653 22834 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:53.173848 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 22697
06:54:53.195 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] handling channelInactive
06:54:53.195 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:53.195 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state READY due to: connection closed
06:54:53.196 [DEBUG - Time-limited test] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:53.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.198 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.205 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:53.205 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:53.205 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [3ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [3ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@156897619(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@e17ac9a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1062e370 -> wakeup thread Time-limited test)) after lookup
06:54:53.206 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:53.207 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.208 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.225 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:53.225 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:53.226 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [3ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [3ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [13ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@156897619(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@e17ac9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1062e370 -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161))) after lookup
06:54:53.227 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:53.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.229 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.264 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:53.265 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:53.265 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [3ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [3ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [13ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [31ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [34ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@156897619(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@e17ac9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1062e370 -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366))) after lookup
06:54:53.266 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:53.267 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.267 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.267 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:53.325 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:53.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=130), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [3ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [3ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [13ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [31ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [34ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [71ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [72ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [126ms] refreshing cache from master, [126ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [129ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@156897619(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@e17ac9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366) -> (continuation of Deferred@936185223 after retry RPC@1720564576), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1062e370 -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366) -> (continuation of Deferred@936185223 after retry RPC@1720564576))) after lookup
06:54:53.326 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:53.328 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.328 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.328 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.745 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:53.745 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:53.746 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=550), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [3ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [3ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [13ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [31ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [34ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [71ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [72ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [126ms] refreshing cache from master, [126ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [129ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [131ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [133ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [133ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [546ms] refreshing cache from master, [546ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [550ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@156897619(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@e17ac9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366) -> (continuation of Deferred@936185223 after retry RPC@1720564576) -> (continuation of Deferred@822374661 after retry RPC@978950910), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1062e370 -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366) -> (continuation of Deferred@936185223 after retry RPC@1720564576) -> (continuation of Deferred@822374661 after retry RPC@978950910))) after lookup
06:54:53.747 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:54:53.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:53.748 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:54:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.058017 23248 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.213763 23263 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:54:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.214226 23263 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:54.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.216195 23263 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:54.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:54.217830 23117 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:54.219915 23117 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:54.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.224025 22992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51" is_pre_election: true
06:54:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.224946 23115 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7; no voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51
06:54:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.225392 23263 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:54:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.486150 23268 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:54:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.486482 23268 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.488473 23268 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:54.490517 22922 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:54.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:54.492589 22922 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.498569 23195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:54:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.499112 23195 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 4.
06:54:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.500262 22920 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51; no voters: 57e078b7be9c46809c193a7d34c36ab3
06:54:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.500732 23268 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Leader pre-election won for term 6
06:54:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.500907 23268 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:54:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.501044 23268 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 5 FOLLOWER]: Advancing to term 6
06:54:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.504222 23268 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.505133 23268 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 6 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.505854 23195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:54:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.506116 23195 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 4 FOLLOWER]: Advancing to term 6
06:54:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:54.507089 22922 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:54:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.510820 23195 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 6.
06:54:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.511427 22920 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51; no voters: 57e078b7be9c46809c193a7d34c36ab3
06:54:54.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.511826 23268 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 FOLLOWER]: Leader election won for term 6
06:54:54.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.512365 23268 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 LEADER]: Becoming Leader. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Running, Role: LEADER
06:54:54.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.512923 23268 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.517271 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 reported cstate change: term changed from 5 to 6, leader changed from 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195) to b006c851bd314c169fdce1bf12a65a51 (127.21.35.194). New cstate: current_term: 6 leader_uuid: "b006c851bd314c169fdce1bf12a65a51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:54:54.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.963625 23195 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Refusing update from remote peer b006c851bd314c169fdce1bf12a65a51: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
06:54:54.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:54.964675 23268 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
06:54:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:54.980335 22922 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 -> Peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Couldn't send request to peer 57e078b7be9c46809c193a7d34c36ab3. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:54:55.725 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:55.726 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:55.726 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2531), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [3ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [3ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [13ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [13ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [31ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [34ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [34ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [71ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [72ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [126ms] refreshing cache from master, [126ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [129ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [131ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [133ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [133ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [546ms] refreshing cache from master, [546ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [550ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [551ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [553ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [553ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2527ms] refreshing cache from master, [2527ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [2530ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@156897619(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@e17ac9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366) -> (continuation of Deferred@936185223 after retry RPC@1720564576) -> (continuation of Deferred@822374661 after retry RPC@978950910) -> (continuation of Deferred@1820092240 after retry RPC@1931766053), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1062e370 -> wakeup thread Time-limited test -> (continuation of Deferred@1775610208 after retry RPC@376111161) -> (continuation of Deferred@627750074 after retry RPC@1808667366) -> (continuation of Deferred@936185223 after retry RPC@1720564576) -> (continuation of Deferred@822374661 after retry RPC@978950910) -> (continuation of Deferred@1820092240 after retry RPC@1931766053))) after lookup
06:54:55.727 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:55.729 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] Successfully connected to peer
06:54:55.751 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2b69b2fb, L:/127.0.0.1:35526 - R:/127.21.35.194:44599] using TOKEN/null
06:54:55.782 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.195:32955
06:54:55.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:54:55.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:55.784761 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:32955
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39365
06:54:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:54:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.166321 23306 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.166684 23306 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.166779 23306 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.175984 23306 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.176199 23306 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.195
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.186931 23306 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:32955
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39365
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.188575 23306 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.190598 23306 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:56.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.206363 23318 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:56.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.206300 23316 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:56.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:56.206692 23315 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:56.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.207185 23306 server_base.cc:1034] running on GCE node
06:54:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.208058 23306 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.209235 23306 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:56.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.210511 23306 hybrid_clock.cc:648] HybridClock initialized: now 1786517696210423 us; error 55 us; skew 500 ppm
06:54:56.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.210820 23306 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:56.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.217286 23306 webserver.cc:460] Webserver started at http://127.21.35.195:39365/ using document root <none> and password file <none>
06:54:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.218113 23306 fs_manager.cc:359] Metadata directory not provided
06:54:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.218242 23306 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:56.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.224823 23306 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
06:54:56.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.228379 23324 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:56.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.230265 23306 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
06:54:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.230508 23306 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.231721 23306 fs_report.cc:383] FS layout report
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/data
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:56.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.253175 23306 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.254144 23306 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.254432 23306 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.255631 23306 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.258244 23332 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:54:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.264839 23306 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:54:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.265035 23306 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:54:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.265187 23306 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:54:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.268879 23306 ts_tablet_manager.cc:610] Registered 1 tablets
06:54:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.269032 23306 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
06:54:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.269455 23332 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap starting.
06:54:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.299528 23332 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Log is configured to *not* fsync() on all Append() calls
06:54:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.332759 23332 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:54:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.333715 23332 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap complete.
06:54:56.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.335441 23332 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent bootstrapping tablet: real 0.066s	user 0.039s	sys 0.024s
06:54:56.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.336162 23306 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.195:32955
06:54:56.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.336189 23447 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.195:32955 every 8 connection(s)
06:54:56.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.337738 23306 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:54:56.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.347126 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 23306
06:54:56.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:56.369 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:56.370 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.194:44599
06:54:56.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:54:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.371258 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 22903
06:54:56.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.383195 23332 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:56.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.385001 23332 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Initialized, Role: FOLLOWER
06:54:56.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.386061 23332 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.389850 23036 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.390965 23449 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.391451 23449 heartbeater.cc:464] Registering TS with master...
06:54:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.392853 23449 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.396426 21694 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.396799 23332 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent starting tablet: real 0.061s	user 0.016s	sys 0.007s
06:54:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.397858 23450 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.398034 23450 heartbeater.cc:464] Registering TS with master...
06:54:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.398042 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.195:53187
06:54:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.398586 23450 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:56.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.400588 21771 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:56.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.403458 23449 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.410465 23448 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.410684 23448 heartbeater.cc:464] Registering TS with master...
06:54:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.411109 23448 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.413229 21846 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:56.505633 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 22903
06:54:56.528 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] handling channelInactive
06:54:56.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:56.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state READY due to: connection closed
06:54:56.530 [DEBUG - Time-limited test] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:56.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.533 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.544 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:56.545 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:56.545 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test)) after lookup
06:54:56.546 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:56.548 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.548 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.548 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.564 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:56.565 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:56.565 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [15ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [18ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [19ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [34ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488))) after lookup
06:54:56.566 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:56.567 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.567 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.567 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.585 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:56.586 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:56.586 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [15ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [18ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [19ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [34ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [37ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [55ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779))) after lookup
06:54:56.587 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:56.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.589 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.605 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:56.606 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:56.606 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [15ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [18ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [19ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [34ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [37ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [55ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [57ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [59ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [60ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [75ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049))) after lookup
06:54:56.607 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:56.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:56.608 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.085 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:57.085 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:57.086 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=556), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [15ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [18ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [19ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [34ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [37ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [55ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [57ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [59ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [60ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [75ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [76ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [78ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [79ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [554ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049) -> (continuation of Deferred@174360944 after retry RPC@1457333818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049) -> (continuation of Deferred@174360944 after retry RPC@1457333818))) after lookup
06:54:57.087 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:57.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.089 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.486 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:57.486 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:54:57.487 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=956), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [15ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [18ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [19ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [34ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [37ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [55ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [57ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [59ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [60ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [75ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [76ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [78ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [79ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [554ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [556ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [559ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [559ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [951ms] refreshing cache from master, [951ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [955ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049) -> (continuation of Deferred@174360944 after retry RPC@1457333818) -> (continuation of Deferred@681404388 after retry RPC@399403067), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049) -> (continuation of Deferred@174360944 after retry RPC@1457333818) -> (continuation of Deferred@681404388 after retry RPC@399403067))) after lookup
06:54:57.488 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:54:57.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.490 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:54:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.563483 23464 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:54:57.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.563787 23464 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:57.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.565596 23464 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:57.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:57.568506 23326 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:57.570751 23326 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.574584 23195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:54:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.575644 23326 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3; no voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51
06:54:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.576189 23464 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
06:54:57.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.890676 23470 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:57.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.890893 23470 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:57.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.891544 23470 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:57.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:57.894538 23115 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:57.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.900561 23399 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3" is_pre_election: true
06:54:57.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.901046 23399 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22bcd60c25a84b47b37128f3341b67b7 in term 5.
06:54:57.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.902189 23117 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:54:57.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.902617 23470 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Leader pre-election won for term 7
06:54:57.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.902779 23470 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Starting leader election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:54:57.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.902882 23470 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 6 FOLLOWER]: Advancing to term 7
06:54:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.905282 23470 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.906081 23470 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 7 election: Requested vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:54:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.906798 23399 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:54:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.907063 23399 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 5 FOLLOWER]: Advancing to term 7
06:54:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:57.908082 23115 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:54:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.911618 23399 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22bcd60c25a84b47b37128f3341b67b7 in term 7.
06:54:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.912204 23117 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:54:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.912544 23470 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Leader election won for term 7
06:54:57.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.913028 23470 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 LEADER]: Becoming Leader. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Running, Role: LEADER
06:54:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.913834 23470 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:57.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:57.918026 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 reported cstate change: term changed from 6 to 7, leader changed from b006c851bd314c169fdce1bf12a65a51 (127.21.35.194) to 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193). New cstate: current_term: 7 leader_uuid: "22bcd60c25a84b47b37128f3341b67b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:54:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:58.468586 23115 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 -> Peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Couldn't send request to peer b006c851bd314c169fdce1bf12a65a51. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:54:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:58.497172 23399 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Refusing update from remote peer 22bcd60c25a84b47b37128f3341b67b7: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
06:54:58.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:58.498167 23490 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
06:54:58.644 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:58.645 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:58.645 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2115), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [3ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [3ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [14ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [15ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [18ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [19ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [34ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [35ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [37ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [37ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [55ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [57ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [59ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [60ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [71ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [75ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [76ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [78ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [79ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [554ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [556ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [559ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [559ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [951ms] refreshing cache from master, [951ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [955ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [958ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [960ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [960ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2111ms] refreshing cache from master, [2111ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [2114ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@619282930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@6b981abf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049) -> (continuation of Deferred@174360944 after retry RPC@1457333818) -> (continuation of Deferred@681404388 after retry RPC@399403067) -> (continuation of Deferred@1899836496 after retry RPC@649255878), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66ad7f14 -> wakeup thread Time-limited test -> (continuation of Deferred@299561176 after retry RPC@478312488) -> (continuation of Deferred@1924304368 after retry RPC@118232779) -> (continuation of Deferred@227149157 after retry RPC@1784148049) -> (continuation of Deferred@174360944 after retry RPC@1457333818) -> (continuation of Deferred@681404388 after retry RPC@399403067) -> (continuation of Deferred@1899836496 after retry RPC@649255878))) after lookup
06:54:58.646 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:58.648 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] Successfully connected to peer
06:54:58.669 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe9e00f7c, L:/127.0.0.1:49572 - R:/127.21.35.193:44873] using TOKEN/null
06:54:58.694 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.194:44599
06:54:58.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:58.695137 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:54:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.048542 23505 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:54:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.048919 23505 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:54:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.049049 23505 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:54:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.058247 23505 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:54:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.058527 23505 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.194
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.070102 23505 tablet_server_runner.cc:78] Tablet server non-default flags:
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:54:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:54:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:54:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.072098 23505 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:54:59.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.074371 23505 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:54:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.092142 23513 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.092171 23514 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:54:59.092228 23516 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:54:59.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.093186 23505 server_base.cc:1034] running on GCE node
06:54:59.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.093878 23505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:54:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.095041 23505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:54:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.096343 23505 hybrid_clock.cc:648] HybridClock initialized: now 1786517699096263 us; error 41 us; skew 500 ppm
06:54:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.096856 23505 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:54:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.099895 23505 webserver.cc:460] Webserver started at http://127.21.35.194:37059/ using document root <none> and password file <none>
06:54:59.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.101084 23505 fs_manager.cc:359] Metadata directory not provided
06:54:59.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.101298 23505 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:54:59.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.108165 23505 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.003s
06:54:59.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.112183 23523 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:54:59.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.113819 23505 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.114068 23505 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.115242 23505 fs_report.cc:383] FS layout report
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/data
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:54:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:54:59.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:54:59.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:54:59.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:54:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.130306 23505 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:54:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.131328 23505 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:54:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.131701 23505 kserver.cc:163] Server-wide thread pool size limit: 3276
06:54:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.132941 23505 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:54:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.135551 23531 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:54:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.142768 23505 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:54:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.142989 23505 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
06:54:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.143153 23505 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:54:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.146879 23505 ts_tablet_manager.cc:610] Registered 1 tablets
06:54:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.147029 23505 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
06:54:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.147535 23531 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap starting.
06:54:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.179610 23531 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Log is configured to *not* fsync() on all Append() calls
06:54:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.216084 23531 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:54:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.220387 23531 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap complete.
06:54:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.220952 23505 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.194:44599
06:54:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.220975 23650 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.194:44599 every 8 connection(s)
06:54:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.222324 23531 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent bootstrapping tablet: real 0.075s	user 0.050s	sys 0.019s
06:54:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.223227 23505 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:54:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.226732 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 23505
06:54:59.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:59.245 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:54:59.245 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.193:44873
06:54:59.246 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:54:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.246483 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 23095
06:54:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.260524 23531 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.262609 23531 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Initialized, Role: FOLLOWER
06:54:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.263384 23531 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:54:59.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.275457 23240 generic_service.cc:196] Checking for leaks (request via RPC)
06:54:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.284303 23652 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:54:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.284725 23652 heartbeater.cc:464] Registering TS with master...
06:54:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.285940 23652 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:54:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.287281 23531 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent starting tablet: real 0.065s	user 0.020s	sys 0.019s
06:54:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.288276 23651 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:54:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.288331 23653 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:54:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.288461 23651 heartbeater.cc:464] Registering TS with master...
06:54:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.288542 23653 heartbeater.cc:464] Registering TS with master...
06:54:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.289110 21692 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.289153 23651 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:54:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.289184 23653 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:54:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.290624 21692 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.194:53829
06:54:59.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.290949 21846 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.292032 21770 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:54:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.294279 23652 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:54:59.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:54:59.402976 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 23095
06:54:59.425 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] handling channelInactive
06:54:59.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state READY due to: connection closed
06:54:59.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:54:59.427 [DEBUG - Time-limited test] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:59.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.429 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.445 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:59.446 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:59.446 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test)) after lookup
06:54:59.447 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:59.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.449 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.465 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:59.465 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:59.466 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [19ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [22ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [22ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [38ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690))) after lookup
06:54:59.466 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:59.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.468 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.484 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:59.484 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:59.485 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [19ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [22ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [22ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [38ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [41ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [41ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [57ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156))) after lookup
06:54:59.485 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:59.487 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.487 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.487 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.605 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:59.605 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:59.605 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=178), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [19ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [22ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [22ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [38ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [41ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [41ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [57ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [58ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [60ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [177ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255))) after lookup
06:54:59.607 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:59.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.608 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.805 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:54:59.805 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:54:59.806 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=378), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [19ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [22ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [22ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [38ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [41ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [41ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [57ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [58ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [60ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [177ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [179ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [181ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [181ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [378ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255) -> (continuation of Deferred@1088841794 after retry RPC@503776571), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255) -> (continuation of Deferred@1088841794 after retry RPC@503776571))) after lookup
06:54:59.806 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:54:59.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:54:59.809 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:00.164 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:00.165 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:00.166 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=738), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [19ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [22ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [22ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [38ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [41ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [41ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [57ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [58ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [60ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [177ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [179ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [181ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [181ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [378ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [379ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [382ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [382ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [734ms] refreshing cache from master, [734ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [737ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255) -> (continuation of Deferred@1088841794 after retry RPC@503776571) -> (continuation of Deferred@280262121 after retry RPC@918324249), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255) -> (continuation of Deferred@1088841794 after retry RPC@503776571) -> (continuation of Deferred@280262121 after retry RPC@918324249))) after lookup
06:55:00.167 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:00.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:00.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:00.168 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:00.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.693430 23667 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 22bcd60c25a84b47b37128f3341b67b7)
06:55:00.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.693645 23667 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:00.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.694288 23667 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:00.697353 23326 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:00.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.704414 23601 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51" is_pre_election: true
06:55:00.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.704913 23601 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 6.
06:55:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.706008 23326 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7
06:55:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.706424 23667 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Leader pre-election won for term 8
06:55:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.706547 23667 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 22bcd60c25a84b47b37128f3341b67b7)
06:55:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.706624 23667 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 7 FOLLOWER]: Advancing to term 8
06:55:00.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.708976 23667 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:00.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.709676 23667 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 8 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.710742 23601 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51"
06:55:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.711027 23601 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 6 FOLLOWER]: Advancing to term 8
06:55:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:00.711582 23326 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.715545 23601 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 8.
06:55:00.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.716221 23326 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7
06:55:00.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.716578 23667 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 FOLLOWER]: Leader election won for term 8
06:55:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.717094 23667 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 LEADER]: Becoming Leader. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Running, Role: LEADER
06:55:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.717676 23667 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:00.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:00.721694 21692 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 reported cstate change: term changed from 7 to 8, leader changed from 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193) to 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195). New cstate: current_term: 8 leader_uuid: "57e078b7be9c46809c193a7d34c36ab3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: HEALTHY } } }
06:55:01.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:01.199023 23326 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 -> Peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Couldn't send request to peer 22bcd60c25a84b47b37128f3341b67b7. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:55:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:01.303660 23601 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Refusing update from remote peer 57e078b7be9c46809c193a7d34c36ab3: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
06:55:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:01.304661 23667 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
06:55:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:02.712692 23326 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111) [suppressed 5 similar messages]
06:55:03.344 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:03.344 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:03.345 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3917), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [2ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [18ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [19ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [22ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [22ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [38ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [39ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [41ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [41ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [57ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [58ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [60ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [177ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [179ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [181ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [181ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [378ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [379ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [382ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [382ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [734ms] refreshing cache from master, [734ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [737ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [739ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [741ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [741ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [3914ms] refreshing cache from master, [3914ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [3917ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1097125860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@25e5b314 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255) -> (continuation of Deferred@1088841794 after retry RPC@503776571) -> (continuation of Deferred@280262121 after retry RPC@918324249) -> (continuation of Deferred@1480080632 after retry RPC@1746019118), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6086a0d5 -> wakeup thread Time-limited test -> (continuation of Deferred@517901508 after retry RPC@1267402690) -> (continuation of Deferred@291534633 after retry RPC@1823924156) -> (continuation of Deferred@1481269021 after retry RPC@47919255) -> (continuation of Deferred@1088841794 after retry RPC@503776571) -> (continuation of Deferred@280262121 after retry RPC@918324249) -> (continuation of Deferred@1480080632 after retry RPC@1746019118))) after lookup
06:55:03.346 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:03.348 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] Successfully connected to peer
06:55:03.365 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x12fd568c, L:/127.0.0.1:34574 - R:/127.21.35.195:32955] using TOKEN/null
06:55:03.398 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.193:44873
06:55:03.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:55:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.399619 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:44873
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35499
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:55:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.775094 23711 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:55:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.775450 23711 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:55:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.775687 23711 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:55:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.784276 23711 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:55:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.784499 23711 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.193
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.794991 23711 tablet_server_runner.cc:78] Tablet server non-default flags:
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:44873
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35499
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:55:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.796517 23711 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:55:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.798496 23711 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:55:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.815405 23718 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.815405 23716 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.816115 23720 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.816392 23711 server_base.cc:1034] running on GCE node
06:55:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.817309 23711 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:55:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.818682 23711 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:55:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.820034 23711 hybrid_clock.cc:648] HybridClock initialized: now 1786517703819924 us; error 93 us; skew 500 ppm
06:55:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.820479 23711 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:55:03.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:03.824929 23326 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 -> Peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Couldn't send request to peer 22bcd60c25a84b47b37128f3341b67b7. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:55:03.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.826702 23711 webserver.cc:460] Webserver started at http://127.21.35.193:35499/ using document root <none> and password file <none>
06:55:03.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.827550 23711 fs_manager.cc:359] Metadata directory not provided
06:55:03.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.827687 23711 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:55:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.834792 23711 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
06:55:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.839763 23728 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:55:03.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.841706 23711 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.000s
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.841897 23711 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:36 on dist-test-slave-mvvj"
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.843151 23711 fs_report.cc:383] FS layout report
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/data
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:55:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:55:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.864274 23711 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:55:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.865164 23711 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:55:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.865522 23711 kserver.cc:163] Server-wide thread pool size limit: 3276
06:55:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.866763 23711 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:55:03.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.869132 23738 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:55:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.875645 23711 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:55:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.875797 23711 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:55:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.875969 23711 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:55:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.879370 23711 ts_tablet_manager.cc:610] Registered 1 tablets
06:55:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.879496 23711 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.000s	sys 0.000s
06:55:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.880266 23738 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap starting.
06:55:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.913661 23738 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Log is configured to *not* fsync() on all Append() calls
06:55:03.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.944093 23711 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.193:44873
06:55:03.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.944195 23866 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.193:44873 every 8 connection(s)
06:55:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.945653 23711 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:55:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.955709 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 23711
06:55:03.957 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:03.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.962256 23738 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:55:03.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.970705 23738 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap complete.
06:55:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.973502 23738 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent bootstrapping tablet: real 0.094s	user 0.072s	sys 0.010s
06:55:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.983342 23738 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.985164 23738 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Initialized, Role: FOLLOWER
06:55:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.986148 23738 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.986639 23870 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:55:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.987008 23870 heartbeater.cc:464] Registering TS with master...
06:55:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.988833 23870 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:55:03.990 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:55:03.992 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.195:32955
06:55:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.992206 21771 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:55:03.992 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:55:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.992946 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 23306
06:55:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.995183 23867 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:55:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.995348 23867 heartbeater.cc:464] Registering TS with master...
06:55:03.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.995960 23867 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:55:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.997654 21847 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:55:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:03.997800 23738 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent starting tablet: real 0.024s	user 0.013s	sys 0.003s
06:55:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.003396 23868 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:55:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.003566 23868 heartbeater.cc:464] Registering TS with master...
06:55:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.004060 23868 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:55:04.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.005544 21694 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:55:04.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.007125 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.193:33051
06:55:04.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.010577 23868 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:55:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.017588 23442 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:04.130883 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 23306
06:55:04.154 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] handling channelInactive
06:55:04.154 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state READY due to: connection closed
06:55:04.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:04.155 [DEBUG - Time-limited test] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:04.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.157 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.165 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:04.165 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:04.165 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@964896246(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@27999934 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11391684 -> wakeup thread Time-limited test)) after lookup
06:55:04.166 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:04.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.169 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.184 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:04.184 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:04.185 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [14ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [14ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@964896246(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@27999934 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11391684 -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921))) after lookup
06:55:04.185 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:04.186 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.186 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.224 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:04.224 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:04.225 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [14ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [14ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [30ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [31ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [31ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@964896246(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@27999934 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11391684 -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502))) after lookup
06:55:04.226 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:04.227 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.228 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.228 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.285 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:04.285 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:04.286 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=131), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [14ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [14ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [30ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [31ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [31ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [70ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [73ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [73ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [130ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@964896246(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@27999934 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502) -> (continuation of Deferred@1778608134 after retry RPC@1979920593), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11391684 -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502) -> (continuation of Deferred@1778608134 after retry RPC@1979920593))) after lookup
06:55:04.287 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:04.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.289 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.504 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:04.505 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:04.505 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=350), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [14ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [14ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [30ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [31ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [31ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [70ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [73ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [73ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [130ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [131ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [134ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [134ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [346ms] refreshing cache from master, [346ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [349ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@964896246(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@27999934 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502) -> (continuation of Deferred@1778608134 after retry RPC@1979920593) -> (continuation of Deferred@1366714268 after retry RPC@502281824), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11391684 -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502) -> (continuation of Deferred@1778608134 after retry RPC@1979920593) -> (continuation of Deferred@1366714268 after retry RPC@502281824))) after lookup
06:55:04.506 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:04.507 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.508 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:04.508 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955
06:55:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.367205 23885 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:55:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.367462 23885 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.369244 23885 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:05.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:05.371544 23733 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:55:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:05.373617 23733 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:55:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.377775 23601 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51" is_pre_election: true
06:55:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.378757 23730 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7; no voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51
06:55:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.379256 23885 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
06:55:05.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.409565 23890 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:55:05.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.409782 23890 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.411267 23890 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:05.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:05.412959 23528 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:55:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:05.414812 23528 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:55:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.420784 23810 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7" is_pre_election: true
06:55:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.421272 23810 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 7.
06:55:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.422408 23525 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51; no voters: 57e078b7be9c46809c193a7d34c36ab3
06:55:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.423041 23890 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Leader pre-election won for term 9
06:55:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.423190 23890 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 57e078b7be9c46809c193a7d34c36ab3)
06:55:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.423275 23890 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 8 FOLLOWER]: Advancing to term 9
06:55:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.425891 23890 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.426633 23890 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 9 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:05.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.427314 23810 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:55:05.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.427654 23810 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 7 FOLLOWER]: Advancing to term 9
06:55:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:05.428501 23528 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111)
06:55:05.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.432395 23810 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b006c851bd314c169fdce1bf12a65a51 in term 9.
06:55:05.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.433027 23525 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, b006c851bd314c169fdce1bf12a65a51; no voters: 57e078b7be9c46809c193a7d34c36ab3
06:55:05.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.433410 23890 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Leader election won for term 9
06:55:05.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.434060 23890 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 LEADER]: Becoming Leader. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Running, Role: LEADER
06:55:05.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.434625 23890 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.438657 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 reported cstate change: term changed from 8 to 9, leader changed from 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195) to b006c851bd314c169fdce1bf12a65a51 (127.21.35.194). New cstate: current_term: 9 leader_uuid: "b006c851bd314c169fdce1bf12a65a51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:55:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.885908 23810 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Refusing update from remote peer b006c851bd314c169fdce1bf12a65a51: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
06:55:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:05.886871 23890 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
06:55:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:05.904558 23528 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 -> Peer 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955): Couldn't send request to peer 57e078b7be9c46809c193a7d34c36ab3. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.195:32955: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:55:06.165 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:06.166 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:55:06.166 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2011), Traces: [0ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [2ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [9ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [11ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [14ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [14ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [29ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [30ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [31ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [31ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [69ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [70ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [73ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [73ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [126ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [130ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [131ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [134ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [134ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [346ms] refreshing cache from master, [346ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [349ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [351ms] sending RPC to server 57e078b7be9c46809c193a7d34c36ab3, [353ms] received response from server 57e078b7be9c46809c193a7d34c36ab3: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [353ms] delaying RPC due to: Network error: Failed to connect to peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955): Connection refused: /127.21.35.195:32955, [2007ms] refreshing cache from master, [2007ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [2010ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@964896246(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@27999934 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502) -> (continuation of Deferred@1778608134 after retry RPC@1979920593) -> (continuation of Deferred@1366714268 after retry RPC@502281824) -> (continuation of Deferred@1106948132 after retry RPC@1571151181), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11391684 -> wakeup thread Time-limited test -> (continuation of Deferred@128023967 after retry RPC@1584776921) -> (continuation of Deferred@1306263579 after retry RPC@760625502) -> (continuation of Deferred@1778608134 after retry RPC@1979920593) -> (continuation of Deferred@1366714268 after retry RPC@502281824) -> (continuation of Deferred@1106948132 after retry RPC@1571151181))) after lookup
06:55:06.167 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:06.168 [DEBUG - kudu-nio-1] (Connection.java:757) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] Successfully connected to peer
06:55:06.186 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x5d8f748d, L:/127.0.0.1:33234 - R:/127.21.35.194:44599] using TOKEN/null
06:55:06.213 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.195:32955
06:55:06.213 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.214493 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:55:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:32955
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39365
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:55:06.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.575598 23923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:55:06.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.576087 23923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:55:06.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.576193 23923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:55:06.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.584555 23923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:55:06.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.584764 23923 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.195
06:55:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.594480 23923 tablet_server_runner.cc:78] Tablet server non-default flags:
06:55:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.195:32955
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.195
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39365
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.195
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/logs
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:55:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:55:06.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.595991 23923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:55:06.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.597872 23923 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:55:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.612571 23931 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:06.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.612594 23932 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:06.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:06.612500 23934 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:06.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.613497 23923 server_base.cc:1034] running on GCE node
06:55:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.614140 23923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:55:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.615381 23923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:55:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.616667 23923 hybrid_clock.cc:648] HybridClock initialized: now 1786517706616595 us; error 38 us; skew 500 ppm
06:55:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.617034 23923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:55:06.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.620249 23923 webserver.cc:460] Webserver started at http://127.21.35.195:39365/ using document root <none> and password file <none>
06:55:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.621241 23923 fs_manager.cc:359] Metadata directory not provided
06:55:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.621446 23923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:55:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.628259 23923 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
06:55:06.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.632045 23942 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:55:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.633867 23923 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.634049 23923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.635324 23923 fs_report.cc:383] FS layout report
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/wal
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/data
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:55:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:55:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:55:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:55:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:55:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:55:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:55:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.665652 23923 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:55:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.666605 23923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:55:06.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.666878 23923 kserver.cc:163] Server-wide thread pool size limit: 3276
06:55:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.667977 23923 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:55:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.670357 23949 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:55:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.676777 23923 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:55:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.676952 23923 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
06:55:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.677070 23923 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:55:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.680518 23923 ts_tablet_manager.cc:610] Registered 1 tablets
06:55:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.680619 23923 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.002s	sys 0.001s
06:55:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.681207 23949 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap starting.
06:55:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.713809 23949 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Log is configured to *not* fsync() on all Append() calls
06:55:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.739032 23923 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.195:32955
06:55:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.739090 24073 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.195:32955 every 8 connection(s)
06:55:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.740695 23923 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-2/data/info.pb
06:55:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.748363 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 23923
06:55:06.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.786319 23949 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:55:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.787333 23949 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Bootstrap complete.
06:55:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.790074 23949 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent bootstrapping tablet: real 0.109s	user 0.068s	sys 0.019s
06:55:06.811 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:55:06.812 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.194:44599
06:55:06.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:55:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.813050 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 23505
06:55:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.825879 23949 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.827638 23949 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Initialized, Role: FOLLOWER
06:55:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.828604 23949 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.830955 23646 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.839948 23949 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3: Time spent starting tablet: real 0.050s	user 0.012s	sys 0.013s
06:55:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.852432 24074 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:55:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.852842 24074 heartbeater.cc:464] Registering TS with master...
06:55:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.853965 24074 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:55:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.854049 24075 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:55:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.854241 24075 heartbeater.cc:464] Registering TS with master...
06:55:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.854889 24075 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:55:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.857790 21847 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.857926 21692 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:06.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.860755 21692 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.195:39359
06:55:06.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.865249 24076 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:55:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.865720 24076 heartbeater.cc:464] Registering TS with master...
06:55:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.865788 24075 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:55:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.866434 24076 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:55:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.868041 21770 ts_manager.cc:194] Re-registered known tserver with Master: 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:06.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:06.954221 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 23505
06:55:06.976 [DEBUG - kudu-nio-1] (Connection.java:239) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] handling channelInactive
06:55:06.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state READY due to: connection closed
06:55:06.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:06.977 [DEBUG - Time-limited test] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:06.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:06.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:06.979 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:06.984 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:06.984 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:55:06.985 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [6ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1087202247(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@60fd4b44 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e6c937d -> wakeup thread Time-limited test)) after lookup
06:55:06.985 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:06.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:06.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:06.986 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.005 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:07.005 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:55:07.005 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [6ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [8ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [9ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [9ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [27ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1087202247(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@60fd4b44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e6c937d -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445))) after lookup
06:55:07.006 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:07.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.009 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.045 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:07.045 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:55:07.045 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [6ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [8ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [9ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [9ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [27ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [29ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [31ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [32ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [67ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1087202247(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@60fd4b44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e6c937d -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163))) after lookup
06:55:07.046 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:07.047 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.047 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.047 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.144 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:07.144 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:55:07.145 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=167), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [6ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [8ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [9ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [9ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [27ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [29ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [31ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [32ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [67ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [69ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [70ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [70ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [166ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1087202247(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@60fd4b44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163) -> (continuation of Deferred@721339311 after retry RPC@961393740), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e6c937d -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163) -> (continuation of Deferred@721339311 after retry RPC@961393740))) after lookup
06:55:07.145 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:07.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.146 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.504 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:07.505 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)[L]]}]
06:55:07.505 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=528), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [6ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [8ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [9ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [9ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [27ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [29ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [31ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [32ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [67ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [69ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [70ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [70ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [166ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [168ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [169ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [169ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [524ms] refreshing cache from master, [524ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [527ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1087202247(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@60fd4b44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163) -> (continuation of Deferred@721339311 after retry RPC@961393740) -> (continuation of Deferred@297443014 after retry RPC@1157740393), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e6c937d -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163) -> (continuation of Deferred@721339311 after retry RPC@961393740) -> (continuation of Deferred@297443014 after retry RPC@1157740393))) after lookup
06:55:07.506 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] connecting to peer
06:55:07.507 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.507 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] cleaning up while in state CONNECTING due to: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:07.507 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599
06:55:08.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.294744 24090 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:55:08.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.295033 24090 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:08.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.295980 24090 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.299659 23730 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:55:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.305693 24020 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3" is_pre_election: true
06:55:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.306349 24020 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22bcd60c25a84b47b37128f3341b67b7 in term 8.
06:55:08.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.307754 23733 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:55:08.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.308224 24090 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Leader pre-election won for term 10
06:55:08.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.308372 24090 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Starting leader election (detected failure of leader b006c851bd314c169fdce1bf12a65a51)
06:55:08.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.308473 24090 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 9 FOLLOWER]: Advancing to term 10
06:55:08.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.310596 24090 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:08.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.311352 24090 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 10 election: Requested vote from peers b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:08.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.312355 24020 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "22bcd60c25a84b47b37128f3341b67b7" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3"
06:55:08.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.312716 24020 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 8 FOLLOWER]: Advancing to term 10
06:55:08.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.312987 23730 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111)
06:55:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.318393 24020 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22bcd60c25a84b47b37128f3341b67b7 in term 10.
06:55:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.318997 23733 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3; no voters: b006c851bd314c169fdce1bf12a65a51
06:55:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.319427 24090 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 10 FOLLOWER]: Leader election won for term 10
06:55:08.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.319952 24090 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 10 LEADER]: Becoming Leader. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Running, Role: LEADER
06:55:08.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.320531 24090 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.324708 21692 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 reported cstate change: term changed from 9 to 10, leader changed from b006c851bd314c169fdce1bf12a65a51 (127.21.35.194) to 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193). New cstate: current_term: 10 leader_uuid: "22bcd60c25a84b47b37128f3341b67b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: UNKNOWN } } }
06:55:08.465 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:08.465 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:08.465 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1488), Traces: [0ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [2ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [2ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [6ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [8ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [9ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [9ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [27ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [29ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [31ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [32ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [67ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [69ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [70ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [70ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [166ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [168ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [169ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [169ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [524ms] refreshing cache from master, [524ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [527ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [528ms] sending RPC to server b006c851bd314c169fdce1bf12a65a51, [530ms] received response from server b006c851bd314c169fdce1bf12a65a51: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [530ms] delaying RPC due to: Network error: Failed to connect to peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599): Connection refused: /127.21.35.194:44599, [1484ms] refreshing cache from master, [1484ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [1487ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@1087202247(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@60fd4b44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163) -> (continuation of Deferred@721339311 after retry RPC@961393740) -> (continuation of Deferred@297443014 after retry RPC@1157740393) -> (continuation of Deferred@1693593363 after retry RPC@398382752), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e6c937d -> wakeup thread Time-limited test -> (continuation of Deferred@2011673307 after retry RPC@581578445) -> (continuation of Deferred@1858975081 after retry RPC@501359163) -> (continuation of Deferred@721339311 after retry RPC@961393740) -> (continuation of Deferred@297443014 after retry RPC@1157740393) -> (continuation of Deferred@1693593363 after retry RPC@398382752))) after lookup
06:55:08.466 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:08.467 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] Successfully connected to peer
06:55:08.486 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4ae04788, L:/127.0.0.1:43740 - R:/127.21.35.193:44873] using TOKEN/null
06:55:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.492584 24020 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Refusing update from remote peer 22bcd60c25a84b47b37128f3341b67b7: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
06:55:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.492736 23730 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 -> Peer b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599): Couldn't send request to peer b006c851bd314c169fdce1bf12a65a51. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.194:44599: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:55:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.493613 24096 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
06:55:08.524 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.194:44599
06:55:08.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:55:08.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.525624 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:08.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:55:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.880486 24120 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:55:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.880851 24120 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:55:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.880946 24120 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:55:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.888955 24120 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:55:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.889158 24120 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.194
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.899096 24120 tablet_server_runner.cc:78] Tablet server non-default flags:
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.194:44599
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.194
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37059
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.194
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/logs
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:55:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.900734 24120 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:55:08.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.902717 24120 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:55:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.917829 24126 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.917790 24127 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:08.917870 24129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.919027 24120 server_base.cc:1034] running on GCE node
06:55:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.919651 24120 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:55:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.920704 24120 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:55:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.921995 24120 hybrid_clock.cc:648] HybridClock initialized: now 1786517708921879 us; error 60 us; skew 500 ppm
06:55:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.922356 24120 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:55:08.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.929747 24120 webserver.cc:460] Webserver started at http://127.21.35.194:37059/ using document root <none> and password file <none>
06:55:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.930588 24120 fs_manager.cc:359] Metadata directory not provided
06:55:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.930707 24120 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:55:08.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.936653 24120 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
06:55:08.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.940157 24135 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:55:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.941854 24120 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.942055 24120 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b006c851bd314c169fdce1bf12a65a51"
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:37 on dist-test-slave-mvvj"
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.943143 24120 fs_report.cc:383] FS layout report
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/wal
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/data
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:55:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:55:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.963797 24120 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:55:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.964727 24120 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:55:08.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.964988 24120 kserver.cc:163] Server-wide thread pool size limit: 3276
06:55:08.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.966053 24120 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:55:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.968484 24142 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:55:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.974985 24120 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:55:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.975104 24120 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
06:55:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.975198 24120 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:55:08.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.978454 24120 ts_tablet_manager.cc:610] Registered 1 tablets
06:55:08.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.978574 24120 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.001s	sys 0.000s
06:55:08.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:08.978940 24142 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap starting.
06:55:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.019294 24142 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Log is configured to *not* fsync() on all Append() calls
06:55:09.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.030702 24120 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.194:44599
06:55:09.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.030735 24268 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.194:44599 every 8 connection(s)
06:55:09.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.032593 24120 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-1/data/info.pb
06:55:09.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.037658 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 24120
06:55:09.038 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:09.071 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:55:09.072 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.21.35.193:44873
06:55:09.072 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:55:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.072824 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 23711
06:55:09.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.107666 23860 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:09.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.115729 24269 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:55:09.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.116180 24269 heartbeater.cc:464] Registering TS with master...
06:55:09.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.117360 24269 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:55:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.119119 24271 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:55:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.119282 24271 heartbeater.cc:464] Registering TS with master...
06:55:09.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.119855 24271 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:55:09.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.120398 21847 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:09.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.121603 21770 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.123085 24270 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:55:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.123248 24270 heartbeater.cc:464] Registering TS with master...
06:55:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.123806 24270 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:55:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.125293 21694 ts_manager.cc:194] Re-registered known tserver with Master: b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.126820 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.194:50435
06:55:09.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.144345 24142 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:55:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.145205 24142 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Bootstrap complete.
06:55:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.146697 24142 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent bootstrapping tablet: real 0.168s	user 0.079s	sys 0.012s
06:55:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.152705 24142 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.153734 24142 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: b006c851bd314c169fdce1bf12a65a51, State: Initialized, Role: FOLLOWER
06:55:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.154616 24142 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.156994 24270 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:55:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.157219 24142 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51: Time spent starting tablet: real 0.010s	user 0.012s	sys 0.001s
06:55:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:09.238245 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 23711
06:55:09.260 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] handling channelInactive
06:55:09.260 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:09.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state READY due to: connection closed
06:55:09.261 [DEBUG - Time-limited test] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.262 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.262 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.262 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.285 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:09.285 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:09.285 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test)) after lookup
06:55:09.286 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.287 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.287 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.304 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:09.305 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:09.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [26ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [26ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [43ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873))) after lookup
06:55:09.306 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.306 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.307 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.307 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:09.325 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:09.326 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [26ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [26ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [43ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [44ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [46ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [46ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [64ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432))) after lookup
06:55:09.327 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.328 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.328 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.328 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.465 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:09.466 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:09.466 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=205), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [26ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [26ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [43ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [44ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [46ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [46ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [64ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [65ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [67ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [67ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [200ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [204ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579))) after lookup
06:55:09.467 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.468 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.685 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:09.685 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:09.685 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=424), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [26ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [26ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [43ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [44ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [46ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [46ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [64ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [65ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [67ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [67ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [200ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [204ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [206ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [207ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [208ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [420ms] refreshing cache from master, [420ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [423ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579) -> (continuation of Deferred@1183579420 after retry RPC@1515125325), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579) -> (continuation of Deferred@1183579420 after retry RPC@1515125325))) after lookup
06:55:09.686 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.687 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.687 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.688 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.904 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:09.905 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)[L],57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955),b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:09.905 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=644), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [26ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [26ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [43ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [44ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [46ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [46ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [64ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [65ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [67ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [67ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [200ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [204ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [206ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [207ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [208ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [420ms] refreshing cache from master, [420ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [423ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [425ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [426ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [427ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [640ms] refreshing cache from master, [640ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [643ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579) -> (continuation of Deferred@1183579420 after retry RPC@1515125325) -> (continuation of Deferred@2126748735 after retry RPC@1408408888), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579) -> (continuation of Deferred@1183579420 after retry RPC@1515125325) -> (continuation of Deferred@2126748735 after retry RPC@1408408888))) after lookup
06:55:09.906 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] connecting to peer
06:55:09.907 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.907 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:09.907 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873) for tablet ca7fe47bd75640edb87d395ae1fa4b98: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873
06:55:10.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.397152 24287 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:55:10.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.397647 24287 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:10.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.399699 24287 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195:32955)
06:55:10.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:10.401084 24137 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:10.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:10.403760 24137 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:10.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.408366 24020 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "b006c851bd314c169fdce1bf12a65a51" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "57e078b7be9c46809c193a7d34c36ab3" is_pre_election: true
06:55:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.409665 24139 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7, 57e078b7be9c46809c193a7d34c36ab3
06:55:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.410210 24287 raft_consensus.cc:2726] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
06:55:10.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.503662 24292 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 22bcd60c25a84b47b37128f3341b67b7)
06:55:10.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.503938 24292 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.505620 24292 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:10.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:10.507212 23944 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:10.509167 23944 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.514901 24218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51" is_pre_election: true
06:55:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.515362 24218 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 9.
06:55:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.517330 23944 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7
06:55:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.517868 24292 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Leader pre-election won for term 11
06:55:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.518028 24292 raft_consensus.cc:482] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 22bcd60c25a84b47b37128f3341b67b7)
06:55:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.518115 24292 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 10 FOLLOWER]: Advancing to term 11
06:55:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.520242 24292 raft_consensus.cc:504] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.520880 24292 leader_election.cc:288] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 11 election: Requested vote from peers 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873), b006c851bd314c169fdce1bf12a65a51 (127.21.35.194:44599)
06:55:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.521878 24218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca7fe47bd75640edb87d395ae1fa4b98" candidate_uuid: "57e078b7be9c46809c193a7d34c36ab3" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "b006c851bd314c169fdce1bf12a65a51"
06:55:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.522100 24218 raft_consensus.cc:3037] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 9 FOLLOWER]: Advancing to term 11
06:55:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:10.522526 23944 leader_election.cc:334] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111)
06:55:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.526281 24218 raft_consensus.cc:2454] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57e078b7be9c46809c193a7d34c36ab3 in term 11.
06:55:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.526796 23944 leader_election.cc:302] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57e078b7be9c46809c193a7d34c36ab3, b006c851bd314c169fdce1bf12a65a51; no voters: 22bcd60c25a84b47b37128f3341b67b7
06:55:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.527207 24292 raft_consensus.cc:2781] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 11 FOLLOWER]: Leader election won for term 11
06:55:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.527709 24292 raft_consensus.cc:686] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [term 11 LEADER]: Becoming Leader. State: Replica: 57e078b7be9c46809c193a7d34c36ab3, State: Running, Role: LEADER
06:55:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.528257 24292 consensus_queue.cc:237] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:10.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:10.532545 21694 catalog_manager.cc:5495] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 reported cstate change: term changed from 10 to 11, leader changed from 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193) to 57e078b7be9c46809c193a7d34c36ab3 (127.21.35.195). New cstate: current_term: 11 leader_uuid: "57e078b7be9c46809c193a7d34c36ab3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } health_report { overall_health: HEALTHY } } }
06:55:10.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:10.929848 23944 consensus_peers.cc:487] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 -> Peer 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873): Couldn't send request to peer 22bcd60c25a84b47b37128f3341b67b7. Status: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:55:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:11.045758 24218 raft_consensus.cc:1261] T ca7fe47bd75640edb87d395ae1fa4b98 P b006c851bd314c169fdce1bf12a65a51 [term 11 FOLLOWER]: Refusing update from remote peer 57e078b7be9c46809c193a7d34c36ab3: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
06:55:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:11.046837 24292 consensus_queue.cc:1035] T ca7fe47bd75640edb87d395ae1fa4b98 P 57e078b7be9c46809c193a7d34c36ab3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
06:55:12.244 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ca7fe47bd75640edb87d395ae1fa4b98 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388' with partition [<start>, <end>)
06:55:12.245 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=ca7fe47bd75640edb87d395ae1fa4b98@[22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873),57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)[L],b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)]}]
06:55:12.245 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=ca7fe47bd75640edb87d395ae1fa4b98, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2984), Traces: [0ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [1ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [1ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [23ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [24ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [26ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [26ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [43ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [44ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [46ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [46ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [60ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [64ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [65ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [67ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [67ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [200ms] refreshing cache from master, [200ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [204ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [206ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [207ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [208ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [420ms] refreshing cache from master, [420ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [423ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [425ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [426ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [427ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [640ms] refreshing cache from master, [640ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [643ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, [645ms] sending RPC to server 22bcd60c25a84b47b37128f3341b67b7, [646ms] received response from server 22bcd60c25a84b47b37128f3341b67b7: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [646ms] delaying RPC due to: Network error: Failed to connect to peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873): Connection refused: /127.21.35.193:44873, [2980ms] refreshing cache from master, [2980ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.35.254:44901, [2983ms] Sub RPC GetTableLocations: received response from server master-127.21.35.254:44901: OK, deferred=Deferred@652955106(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$83/1985374328@175ef0f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579) -> (continuation of Deferred@1183579420 after retry RPC@1515125325) -> (continuation of Deferred@2126748735 after retry RPC@1408408888) -> (continuation of Deferred@1865168157 after retry RPC@1983943403), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ab868b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1016167362 after retry RPC@1426415873) -> (continuation of Deferred@1725463541 after retry RPC@1508289432) -> (continuation of Deferred@430544413 after retry RPC@1979663579) -> (continuation of Deferred@1183579420 after retry RPC@1515125325) -> (continuation of Deferred@2126748735 after retry RPC@1408408888) -> (continuation of Deferred@1865168157 after retry RPC@1983943403))) after lookup
06:55:12.246 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] connecting to peer
06:55:12.247 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] Successfully connected to peer
06:55:12.266 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9584575c, L:/127.0.0.1:58392 - R:/127.21.35.195:32955] using TOKEN/null
06:55:12.292 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.21.35.193:44873
06:55:12.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.293921 21647 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:44873
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:55:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35499
06:55:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:55:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:55:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.656920 24337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:55:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.657267 24337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:55:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.657395 24337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:55:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.665127 24337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:55:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.665411 24337 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.35.193
06:55:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.676241 24337 tablet_server_runner.cc:78] Tablet server non-default flags:
06:55:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:55:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.35.212:37137
06:55:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.35.193:44873
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.35.193
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35499
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.35.254:44901,127.21.35.253:33133,127.21.35.252:34003
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.35.193
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/logs
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:55:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:55:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 12 Aug 2026 05:55:59 UTC on ccycloud.quasar-tbxajw.root.comops.site
06:55:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581995
06:55:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:55:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.678210 24337 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:55:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.680226 24337 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:55:12.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.695047 24343 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:12.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.695791 24344 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:12.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:12.697875 24346 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:55:12.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.698859 24337 server_base.cc:1034] running on GCE node
06:55:12.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.699651 24337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:55:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.700904 24337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:55:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.702242 24337 hybrid_clock.cc:648] HybridClock initialized: now 1786517712702081 us; error 113 us; skew 500 ppm
06:55:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.702704 24337 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:55:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.705910 24337 webserver.cc:460] Webserver started at http://127.21.35.193:35499/ using document root <none> and password file <none>
06:55:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.706791 24337 fs_manager.cc:359] Metadata directory not provided
06:55:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.706992 24337 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:55:12.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.714003 24337 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
06:55:12.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.718489 24354 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:55:12.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.720508 24337 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.000s
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.720715 24337 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data,/tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22bcd60c25a84b47b37128f3341b67b7"
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:54:36 on dist-test-slave-mvvj"
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.722009 24337 fs_report.cc:383] FS layout report
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/wal
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/data
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:55:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:55:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.756130 24337 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
06:55:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.757200 24337 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:55:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.757498 24337 kserver.cc:163] Server-wide thread pool size limit: 3276
06:55:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.758684 24337 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:55:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.761039 24362 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
06:55:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.767094 24337 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:55:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.767246 24337 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
06:55:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.767380 24337 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
06:55:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.770937 24337 ts_tablet_manager.cc:610] Registered 1 tablets
06:55:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.771026 24337 ts_tablet_manager.cc:589] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
06:55:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.771482 24362 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap starting.
06:55:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.802170 24362 log.cc:824] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Log is configured to *not* fsync() on all Append() calls
06:55:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.827272 24337 rpc_server.cc:307] RPC server started. Bound to: 127.21.35.193:44873
06:55:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.827431 24481 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.35.193:44873 every 8 connection(s)
06:55:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.829061 24337 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoM2LY4/test-tmp/mini-kudu-cluster7745568943443226433/ts-0/data/info.pb
06:55:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.837236 21647 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu as pid 24337
06:55:12.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:55:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.865535 24482 heartbeater.cc:347] Connected to a master server at 127.21.35.252:34003
06:55:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.866066 24482 heartbeater.cc:464] Registering TS with master...
06:55:12.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.868119 24482 heartbeater.cc:511] Master 127.21.35.252:34003 requested a full tablet report, sending...
06:55:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.871744 21847 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:55:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.873205 24483 heartbeater.cc:347] Connected to a master server at 127.21.35.254:44901
06:55:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.873389 24483 heartbeater.cc:464] Registering TS with master...
06:55:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.874095 24483 heartbeater.cc:511] Master 127.21.35.254:44901 requested a full tablet report, sending...
06:55:12.874 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:55:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.876175 24484 heartbeater.cc:347] Connected to a master server at 127.21.35.253:33133
06:55:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.876309 24484 heartbeater.cc:464] Registering TS with master...
06:55:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.876912 24484 heartbeater.cc:511] Master 127.21.35.253:33133 requested a full tablet report, sending...
06:55:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.878129 21694 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:55:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.879767 21694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.35.193:46453
06:55:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.881842 21770 ts_manager.cc:194] Re-registered known tserver with Master: 22bcd60c25a84b47b37128f3341b67b7 (127.21.35.193:44873)
06:55:12.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.884270 24362 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:55:12.885 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786517678388, tablet=ca7fe47bd75640edb87d395ae1fa4b98, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:55:12.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.885711 24362 tablet_bootstrap.cc:492] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Bootstrap complete.
06:55:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.887336 24362 ts_tablet_manager.cc:1397] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent bootstrapping tablet: real 0.116s	user 0.062s	sys 0.027s
06:55:12.895 [DEBUG - main] (Connection.java:575) [peer master-127.21.35.254:44901(127.21.35.254:44901)] disconnecting while in state READY
06:55:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.896410 24362 raft_consensus.cc:348] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:12.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.897946 24362 raft_consensus.cc:729] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22bcd60c25a84b47b37128f3341b67b7, State: Initialized, Role: FOLLOWER
06:55:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.898931 24362 consensus_queue.cc:260] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22bcd60c25a84b47b37128f3341b67b7" member_type: VOTER last_known_addr { host: "127.21.35.193" port: 44873 } } peers { permanent_uuid: "b006c851bd314c169fdce1bf12a65a51" member_type: VOTER last_known_addr { host: "127.21.35.194" port: 44599 } } peers { permanent_uuid: "57e078b7be9c46809c193a7d34c36ab3" member_type: VOTER last_known_addr { host: "127.21.35.195" port: 32955 } }
06:55:12.900 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.35.254:44901(127.21.35.254:44901)] handling channelInactive
06:55:12.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.35.254:44901(127.21.35.254:44901)] cleaning up while in state READY due to: connection closed
06:55:12.905 [DEBUG - main] (Connection.java:575) [peer master-127.21.35.252:34003(127.21.35.252:34003)] disconnecting while in state READY
06:55:12.906 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.35.252:34003(127.21.35.252:34003)] handling channelInactive
06:55:12.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.35.252:34003(127.21.35.252:34003)] cleaning up while in state READY due to: connection closed
06:55:12.907 [DEBUG - main] (Connection.java:575) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] disconnecting while in state READY
06:55:12.908 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] handling channelInactive
06:55:12.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57e078b7be9c46809c193a7d34c36ab3(127.21.35.195:32955)] cleaning up while in state READY due to: connection closed
06:55:12.909 [DEBUG - main] (Connection.java:575) [peer b006c851bd314c169fdce1bf12a65a51(127.21.35.194:44599)] disconnecting while in state TERMINATED
06:55:12.909 [DEBUG - main] (Connection.java:575) [peer master-127.21.35.253:33133(127.21.35.253:33133)] disconnecting while in state READY
06:55:12.910 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.35.253:33133(127.21.35.253:33133)] handling channelInactive
06:55:12.910 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.35.253:33133(127.21.35.253:33133)] cleaning up while in state READY due to: connection closed
06:55:12.910 [DEBUG - main] (Connection.java:575) [peer 22bcd60c25a84b47b37128f3341b67b7(127.21.35.193:44873)] disconnecting while in state TERMINATED
06:55:12.912 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:55:12.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.914554 24362 ts_tablet_manager.cc:1428] T ca7fe47bd75640edb87d395ae1fa4b98 P 22bcd60c25a84b47b37128f3341b67b7: Time spent starting tablet: real 0.027s	user 0.018s	sys 0.007s
06:55:12.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.914870 24483 heartbeater.cc:503] Master 127.21.35.254:44901 was elected leader, sending a full tablet report...
06:55:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.916785 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 24337
06:55:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:12.941478 24477 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.040995 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 24337
06:55:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.072867 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 24120
06:55:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:55:13.076565 24499 negotiation.cc:337] Failed RPC negotiation. Trace:
06:55:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:55:13.050205 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.21.35.193:44873 (local address 127.21.35.195:53947)
06:55:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:55:13.050906 (+   701us) negotiation.cc:107] Waiting for socket to connect
06:55:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:55:13.050926 (+    20us) client_negotiation.cc:174] Beginning negotiation
06:55:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:55:13.051058 (+   132us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
06:55:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:55:13.072198 (+ 21140us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.21.35.193:44873: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:55:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":561,"thread_start_us":376,"threads_started":1}
06:55:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.085953 24264 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.198787 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 24120
06:55:13.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.222716 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 23923
06:55:13.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.233170 24068 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.363507 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 23923
06:55:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.388872 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 21659
06:55:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.390543 21728 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.520884 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 21659
06:55:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.549161 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 21738
06:55:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.550326 21803 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.671597 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 21738
06:55:13.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.699409 21647 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu pid 21810
06:55:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.700723 21881 generic_service.cc:196] Checking for leaks (request via RPC)
06:55:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:55:13.821254 21647 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoM2LY4/build/asan/bin/kudu with pid 21810
06:55:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T06:55:13Z chronyd exiting

Time: 41.29

OK (1 test)

